Preparing for 0.2.3 release

Grst master

.github/workflows/pr-checks.yaml

@@ -0,0 +1,49 @@
+# Tests that run on every PR
+
+name: Pull Request Checks
+
+on: [push, pull_request]
+
+jobs:
+  black:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+        name: Checkout repository
+
+      - uses: actions/setup-python@v2
+        name: Setup Python
+        with:
+          python-version: 3.9
+
+      - name: Install black
+        run: |
+          python -m pip install --upgrade pip
+          pip install black
+      - name: Run black
+        run: |
+          black -l 79 . --check
+  # This job is copied over from `deploy.yaml`
+  run-tests:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+
+      # See: https://github.com/marketplace/actions/setup-conda
+      - uses: s-weigand/setup-conda@v1
+        with:
+          conda-channels: "conda-forge"
+
+      - name: Build environment
+        run: |
+          conda env create -f environment.yml
+          eval "$(conda shell.bash hook)"
+          conda activate cellxgene-gateway
+          python setup.py install
+
+      - name: Run tests
+        run: |
+          eval "$(conda shell.bash hook)"
+          conda activate cellxgene-gateway
+          python -m unittest discover tests
+          bash <(curl -s https://codecov.io/bash)

Changelog.md

@@ -1,3 +1,7 @@
+# 0.2.3
+
+* Added support for ProxyFix
+
 # 0.2.2
 
   * Fixed bug with annotations (missing annotation.js asset)

README.md

@@ -60,6 +60,7 @@ Here's what the environment variables mean:
 
 * `CELLXGENE_LOCATION` - the location of the cellxgene executable, e.g. `~/anaconda2/envs/cellxgene/bin/cellxgene`
 * `CELLXGENE_DATA` - a directory that can contain subdirectories with `.h5ad` data files, *without* trailing slash, e.g. `/mnt/cellxgene_data`
+
 Optional environment variables:
 * `CELLXGENE_ARGS` - catch-all variable that can be used to pass additional command line args to cellxgene server
 * `EXTERNAL_HOST` - the hostname and port from the perspective of the web browser, typically `localhost:5005` if running locally. Defaults to "localhost:{GATEWAY_PORT}"
@@ -69,7 +70,14 @@ Optional environment variables:
 * `GATEWAY_EXTRA_SCRIPTS` - JSON array of script paths, will be embedded into each page and forwarded with `--scripts` to cellxgene server
 * `GATEWAY_ENABLE_UPLOAD` - Set to `true` or `1` to enable HTTP uploads. This is not recommended for a public server.
 * `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations. 
-* `GATEWAY_ENABLE_BACKED_MODE` - Set to `true` or to `1` to load AnnData in file-backed mode. This saves memory and speeds up launch time but may reduce overall performance.  
+* `GATEWAY_ENABLE_BACKED_MODE` - Set to `true` or to `1` to load AnnData in file-backed mode. This saves memory and speeds up launch time but may reduce overall performance.
+
+If any of the following optional variables are set, [ProxyFix](https://werkzeug.palletsprojects.com/en/1.0.x/middleware/proxy_fix/) will be used.
+* `PROXY_FIX_FOR` - Number of upstream proxies setting X-Forwarded-For
+* `PROXY_FIX_PROTO` - Number of upstream proxies setting X-Forwarded-Proto
+* `PROXY_FIX_HOST` - Number of upstream proxies setting X-Forwarded-Host
+* `PROXY_FIX_PORT` - Number of upstream proxies setting X-Forwarded-Port
+* `PROXY_FIX_PREFIX` - Number of upstream proxies setting X-Forwarded-Prefix
 
 The defaults should be fine if you set up a venv and cellxgene_data folder as above.
 
@@ -113,6 +121,14 @@ python setup.py develop
 
 For convenience, the code repo includes a `run.sh.example` shell script to run the gateway.
 
+4. Install pre-commit hooks
+
+```bash
+conda install -c conda-forge pre-commit
+pre-commit install
+```
+
+
 ## Running Tests
 
 [![Build Status](https://travis-ci.org/Novartis/cellxgene-gateway.svg?branch=master)](https://travis-ci.org/Novartis/cellxgene-gateway)

cellxgene_gateway/__init__.py

@@ -7,4 +7,4 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
-__version__ = "0.2.2"
+__version__ = "0.2.3"

cellxgene_gateway/cache_entry.py

@@ -8,6 +8,8 @@
 # the specific language governing permissions and limitations under the License.
 import datetime
 import logging
+from flask.helpers import url_for
+from flask.wrappers import Response
 
 import psutil
 from enum import Enum
@@ -120,7 +122,7 @@ class CacheEntry:
         return gateway_content
 
     def gateway_basepath(self):
-        return f"{env.external_protocol}://{env.external_host}/view/{self.key.pathpart}/"
+        return url_for("do_view", path=self.key.pathpart) + "/"
 
     def cellxgene_basepath(self):
         return f"http://127.0.0.1:{self.port}"
@@ -130,7 +132,7 @@ class CacheEntry:
         subpath = path[len(self.key.pathpart) :]  # noqa: E203
 
         if len(subpath) == 0:
-            r = make_response(f"Redirect to {gateway_basepath}\n", 301)
+            r = make_response(f"Redirect to {gateway_basepath}\n", 302)
             r.headers["location"] = gateway_basepath + querystring()
             return r
         elif self.status == CacheEntryStatus.loading:

cellxgene_gateway/env.py

@@ -47,6 +47,12 @@ env_vars = {
     "CELLXGENE_DATA": cellxgene_data,
 }
 
+proxy_fix_for = int(os.environ.get("PROXY_FIX_FOR", "0"))
+proxy_fix_proto = int(os.environ.get("PROXY_FIX_PROTO", "0"))
+proxy_fix_host = int(os.environ.get("PROXY_FIX_HOST", "0"))
+proxy_fix_port = int(os.environ.get("PROXY_FIX_PORT", "0"))
+proxy_fix_prefix = int(os.environ.get("PROXY_FIX_PREFIX", "0"))
+
 optional_env_vars = {
     "EXTERNAL_HOST": external_host,
     "EXTERNAL_PROTOCOL": external_protocol,
@@ -58,6 +64,11 @@ optional_env_vars = {
     "GATEWAY_ENABLE_ANNOTATIONS": enable_annotations,
     "GATEWAY_ENABLE_BACKED_MODE": enable_backed_mode,
     "CELLXGENE_ARGS": cellxgene_args,
+    "PROXY_FIX_FOR": proxy_fix_for,
+    "PROXY_FIX_PROTO": proxy_fix_proto,
+    "PROXY_FIX_HOST": proxy_fix_host,
+    "PROXY_FIX_PORT": proxy_fix_port,
+    "PROXY_FIX_PREFIX": proxy_fix_prefix,
 }
 
 

cellxgene_gateway/filecrawl.py

@@ -14,6 +14,7 @@ from cellxgene_gateway.dir_util import (
     make_annotations,
     annotations_suffix,
 )
+from flask import url_for
 
 
 def recurse_dir(path):
@@ -91,7 +92,7 @@ def render_entries(entries):
 
 
 def get_url(entry):
-    return f"/view/{ entry['path'].lstrip('/') }"
+    return url_for("do_view", path=entry["path"].lstrip("/") + "/")
 
 
 def get_class(entry):

cellxgene_gateway/gateway.py

@@ -25,7 +25,7 @@ from flask import (
 )
 from flask_api import status
 from werkzeug.utils import secure_filename
-
+from werkzeug.middleware.proxy_fix import ProxyFix
 from cellxgene_gateway import env
 from cellxgene_gateway.backend_cache import BackendCache
 from cellxgene_gateway.cache_entry import CacheEntryStatus
@@ -50,9 +50,23 @@ def _force_https(app):
 
 
 app.wsgi_app = _force_https(app.wsgi_app)
+if (
+    env.proxy_fix_for > 0
+    or env.proxy_fix_proto > 0
+    or env.proxy_fix_host > 0
+    or env.proxy_fix_port > 0
+    or env.proxy_fix_prefix > 0
+):
+    app.wsgi_app = ProxyFix(
+        app.wsgi_app,
+        x_for=env.proxy_fix_for,
+        x_proto=env.proxy_fix_proto,
+        x_host=env.proxy_fix_host,
+        x_port=env.proxy_fix_port,
+        x_prefix=env.proxy_fix_prefix,
+    )
 
 cache = BackendCache()
-location = f"{env.external_protocol}://{env.external_host}"
 
 
 @app.errorhandler(CellxgeneException)
@@ -126,7 +140,7 @@ def make_user():
 
     create_dir(env.cellxgene_data, dir_name)
 
-    return redirect(location, code=302)
+    return redirect(url_for("index"), code=302)
 
 
 def make_subdir():
@@ -135,7 +149,7 @@ def make_subdir():
 
     create_dir(parent_path, dir_name)
 
-    return redirect(location, code=302)
+    return redirect(url_for("index"), code=302)
 
 
 def upload_file():
@@ -154,7 +168,7 @@ def upload_file():
                             full_upload_path, secure_filename(f.filename)
                         )
                     )
-                    return redirect("/filecrawl.html", code=302)
+                    return redirect(url_for("filecrawl"), code=302)
                 else:
                     raise CellxgeneException(
                         "Uploaded file must be in anndata (.h5ad) format.",
@@ -170,7 +184,7 @@ def upload_file():
             "Invalid directory.", status.HTTP_400_BAD_REQUEST
         )
 
-    return redirect(location, code=302)
+    return redirect(url_for("index"), code=302)
 
 
 if env.enable_upload:

cellxgene_gateway/prune_process_cache.py

@@ -10,14 +10,16 @@
 import logging
 import time
 
-from cellxgene_gateway.env import ttl
-from cellxgene_gateway.util import current_time_stamp
+from cellxgene_gateway import env
+from cellxgene_gateway import util
+
+logger = logging.getLogger(__name__)
 
 
 class PruneProcessCache:
     def __init__(self, cache):
         self.cache = cache
-        self.expire_seconds = 3600 if ttl is None else int(ttl)
+        self.expire_seconds = 3600 if env.ttl is None else int(env.ttl)
 
     def __call__(self):
         while True:
@@ -25,7 +27,7 @@ class PruneProcessCache:
             self.prune()
 
     def prune(self):
-        timestamp = current_time_stamp()
+        timestamp = util.current_time_stamp()
         cutoff = timestamp - self.expire_seconds
         processes_to_delete = [
             p for p in self.cache.entry_list if p.timestamp < cutoff
@@ -33,9 +35,9 @@ class PruneProcessCache:
         processes_to_keep = [
             p for p in self.cache.entry_list if not p.timestamp < cutoff
         ]
-        logger = logging.getLogger("cellxgene_gateway")
+
         logger.debug(
-            f"Cutoff {cutoff} = timestamp {timestamp} - expire seconds {self.expire_seconds} , keeping {processes_to_keep}"
+            f"Cutoff {cutoff} = timestamp {timestamp} - expire seconds {self.expire_seconds} , keeping {processes_to_keep}, pruning {processes_to_delete}"
         )
 
         for process in processes_to_delete:

cellxgene_gateway/templates/cellxgene_error.html

@@ -28,11 +28,11 @@
           
      <h4>{{ message }}</h4>
 
-     <a href="/filecrawl.html">
+     <a href="{{ url_for('filecrawl') }}">
           Please click here to be redirected to the file directory.
      </a>
      <br>
-     <a href="/">
+     <a href="{{ url_for('index') }}">
           Please click here to return to the homepage.
      </a>
      </div>

cellxgene_gateway/templates/filecrawl.html

@@ -36,10 +36,10 @@
           Navigation:
      <ul>
      {% if path %}
-     <li><a href="/filecrawl.html">top level</a></li>
+     <li><a href="{{ url_for('filecrawl') }}">top level</a></li>
      {% else %}
      {% endif %}
-     <li><a href="/">homepage</a></li>
+     <li><a href="{{ url_for('index') }}">homepage</a></li>
      </ul>
      </p>
      <script>

cellxgene_gateway/templates/index.html

@@ -35,12 +35,12 @@
 		Links:	
 	</h1>
 	<div class="list-group" style="width:50%;padding-left:65px">
-	  <a href="/filecrawl.html" class="list-group-item list-group-item-action">
+	  <a href="{{ url_for('filecrawl') }}" class="list-group-item list-group-item-action">
 		  <u>File Crawler: Allows you to view all uploaded data.</u></a>
 		  
 	</div>
 	<div class="list-group" style="width:50%;padding-left:65px">
-		<a href="/cache_status" class="list-group-item list-group-item-action">
+		<a href="{{ url_for('do_GET_status') }}" class="list-group-item list-group-item-action">
 			<u>Cache Status: view status of launched cellxgene servers.</u></a>
 	</div>
 	

cellxgene_gateway/templates/loading.html

@@ -35,11 +35,11 @@
           The page will refresh shortly.
      </p>
 
-     <a href="/filecrawl.html">
+     <a href="{{ url_for('filecrawl') }}">
           Please click here to be redirected to the file directory.
      </a>
      <br>
-     <a href="/">
+     <a href="{{ url_for('index') }}">
           Please click here to return to the homepage.
      </a>
      </div>

cellxgene_gateway/templates/process_error.html

@@ -39,10 +39,10 @@
           <li><a href="{{url_for('do_relaunch', path=dataset)}}">
                     Attempt to relaunch the cellxgene server.
                </a></li>
-          <li><a href="/filecrawl.html">
+          <li><a href="{{ url_for('filecrawl') }}">
                Return to the file directory.
           </a></li>
-          <li><a href="/">
+          <li><a href="{{ url_for('index') }}">
                Return to the homepage.
           </a></li>
      </ul>

tests/test_cache_entry.py

@@ -1,11 +1,19 @@
 import unittest
+from flask import Flask
 from cellxgene_gateway.cache_entry import CacheEntry, CacheEntryStatus
 from cellxgene_gateway.cache_key import CacheKey
+from cellxgene_gateway.gateway import app
 
 key = CacheKey("czi/pbmc3k.h5ad", "pbmc3k.h5ad", "tmp.csv")
 
 
 class TestRenderEntry(unittest.TestCase):
+    def setUp(self):
+        self.app = app
+        self.app_context = self.app.test_request_context()
+        self.app_context.push()
+        self.client = self.app.test_client()
+
     def test_GIVEN_key_and_port_THEN_returns_loading_CacheEntry(self):
         entry = CacheEntry.for_key("some-key", 1)
         self.assertEqual(entry.status, CacheEntryStatus.loading)
@@ -14,16 +22,14 @@ class TestRenderEntry(unittest.TestCase):
         actual = CacheEntry.for_key(key, 8000).rewrite_text_content(
             "src:url(/static/assets/"
         )
-        expected = (
-            "src:url(http://localhost:5005/view/czi/pbmc3k.h5ad/static/assets/"
-        )
+        expected = "src:url(/view/czi/pbmc3k.h5ad/static/assets/"
         self.assertEqual(actual, expected)
 
     def test_GIVEN_absolute_src_THEN_include_path(self):
         actual = CacheEntry.for_key(key, 8000).rewrite_text_content(
             '<link rel="shortcut icon" href="/static/assets/favicon.ico">'
         )
-        expected = '<link rel="shortcut icon" href="http://localhost:5005/view/czi/pbmc3k.h5ad/static/assets/favicon.ico">'
+        expected = '<link rel="shortcut icon" href="/view/czi/pbmc3k.h5ad/static/assets/favicon.ico">'
         self.assertEqual(actual, expected)