prepared for 0.2.1 release

updated docs
made GATEWAY_IP optional

.gitignore

@@ -136,3 +136,6 @@ dmypy.json
 .pyre/
 
 # End of https://www.gitignore.io/api/python
+
+*.patch
+.vscode

.travis.yml

@@ -0,0 +1,41 @@
+# This is necessary for nxviz as matplotlib is involved.
+# before_script:
+#   - "export DISPLAY=:99.0"
+#   - "sh -e /etc/init.d/xvfb start"
+#   - sleep 5 # give xvfb some time to start
+
+language: python
+matrix:
+  include:
+    - python: 3.5  # we don't actually use this
+      env: PYTHON_VERSION=3.7
+
+install:
+  # We do this conditionally because it saves us some downloading if the
+  # version is the same.
+  - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
+  - bash miniconda.sh -b -p $HOME/miniconda
+  - export PATH="$HOME/miniconda/bin:$PATH"
+  - hash -r
+  - conda config --set always_yes yes --set changeps1 no
+  - conda update -q conda
+  - conda config --add channels conda-forge
+
+  # Useful for debugging any issues with conda
+  - conda info -a
+
+  # Install Python, py.test, and required packages.
+  - conda env create -f environment.yml
+  - source activate cellxgene-dev
+  - python setup.py install
+
+script:
+  # Your test script goes here
+  - black -l 79 . --check
+  - python -m unittest discover tests
+
+after_success:
+  - bash <(curl -s https://codecov.io/bash)
+
+notifications:
+  email: true

Changelog.md

@@ -0,0 +1,13 @@
+# 0.2.1
+
+  * Minor fixes to enable cellxgene 0.16.0
+  * Added CELLXGENE_ARGS to enable passing additional arguments to cellxgene
+  * added metadata/ip_address endpoint
+
+# 0.2.0
+
+Incrementing minor version since the changes for 0.15 are breaking, and we may want to release bugfixes from 0.1.0 branch.
+
+# 0.1.1
+
+Added support for cellxgene 0.15

README.md

@@ -30,7 +30,7 @@ Note: you may need to downgrade h5py with `pip install h5py==2.9.0` due to an [i
 ### Option 2: Install from PyPI
 
 ```bash
-# NOT YET DONE, COMING! STAY TUNED
+pip install cellxgene-gateway
 ```
 
 ## Running cellxgene gateway
@@ -39,7 +39,7 @@ Note: you may need to downgrade h5py with `pip install h5py==2.9.0` due to an [i
 
 ```bash
 mkdir ../cellxgene_data
-wget https://github.com/chanzuckerberg/cellxgene/raw/master/example-dataset/pbmc3k.h5ad -O ../cellxgene_data/pbmc3k.h5ad
+wget https://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/example-dataset/pbmc3k.h5ad -O ../cellxgene_data/pbmc3k.h5ad
 ```
 
 
@@ -61,12 +61,15 @@ Here's what the environment variables mean:
 * `CELLXGENE_LOCATION` - the location of the cellxgene executable, e.g. `~/anaconda2/envs/cellxgene/bin/cellxgene`
 * `CELLXGENE_DATA` - a directory that can contain subdirectories with `.h5ad` data files, *without* trailing slash, e.g. `/mnt/cellxgene_data`
 Optional environment variables:
+* `CELLXGENE_ARGS` - catch-all variable that can be used to pass additional command line args to cellxgene server
 * `EXTERNAL_HOST` - the hostname and port from the perspective of the web browser, typically `localhost:5005` if running locally. Defaults to "localhost:{GATEWAY_PORT}"
 * `EXTERNAL_PROTOCOL` - typically http when running locally, can be https when deployed if the gateway is behind a load balancer or reverse proxy that performs https termination. Default value "http"
 * `GATEWAY_IP` - ip addess of instance gateway is running on, mostly used to display SSH instructions. Defaults to `socket.gethostbyname(socket.gethostname())`
 * `GATEWAY_PORT` - local port that the gateway should bind to, defaults to 5005
 * `GATEWAY_EXTRA_SCRIPTS` - JSON array of script paths, will be embedded into each page and forwarded with `--scripts` to cellxgene server
 * `GATEWAY_ENABLE_UPLOAD` - Set to `true` or `1` to enable HTTP uploads. This is not recommended for a public server.
+* `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations. 
+* `GATEWAY_ENABLE_BACKED_MODE` - Set to `true` or to `1` to load AnnData in file-backed mode. This saves memory and speeds up launch time but may reduce overall performance.  
 
 The defaults should be fine if you set up a venv and cellxgene_data folder as above.
 
@@ -112,6 +115,8 @@ For convenience, the code repo includes a `run.sh.example` shell script to run t
 
 ## Running Tests
 
+[![Build Status](https://travis-ci.org/Novartis/cellxgene-gateway.svg?branch=master)](https://travis-ci.org/Novartis/cellxgene-gateway)
+
 ```bash
     python -m unittest discover tests
 ```

cellxgene_gateway/__init__.py

@@ -6,3 +6,5 @@
 # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
+
+__version__ = "0.2.1"

cellxgene_gateway/backend_cache.py

@@ -22,8 +22,10 @@ process_backend = SubprocessBackend()
 
 def is_port_in_use(port):
     import socket
+
     with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-        return s.connect_ex(('localhost', port)) == 0
+        return s.connect_ex(("localhost", port)) == 0
+
 
 class BackendCache:
     def __init__(self):
@@ -38,7 +40,9 @@ class BackendCache:
         matches = [
             c
             for c in contents
-            if c.key.dataset == key.dataset and c.key.annotation_file == key.annotation_file and c.status != "terminated"
+            if c.key.dataset == key.dataset
+            and c.key.annotation_file == key.annotation_file
+            and c.status != "terminated"
         ]
 
         if len(matches) == 0:

cellxgene_gateway/cache_entry.py

@@ -12,12 +12,14 @@ import datetime
 
 from flask import make_response, request, render_template
 from requests import get, post, put
+import re
 
 from cellxgene_gateway import env
 from cellxgene_gateway.cellxgene_exception import CellxgeneException
 from cellxgene_gateway.util import current_time_stamp
 from cellxgene_gateway.flask_util import querystring
 
+
 class CacheEntry:
     def __init__(
         self,
@@ -79,8 +81,10 @@ class CacheEntry:
         pid = self.pid
         if pid != None and self.status != "terminated":
             terminated = []
+
             def on_terminate(p):
                 terminated.append(p.pid)
+
             p = psutil.Process(pid)
             children = p.children()
             for child in children:
@@ -89,66 +93,79 @@ class CacheEntry:
             terminated.append(p.pid)
             p.terminate()
             psutil.wait_procs([p], callback=on_terminate)
-            logging.getLogger("cellxgene_gateway").info(f"terminated {terminated}")
+            logging.getLogger("cellxgene_gateway").info(
+                f"terminated {terminated}"
+            )
         self.status = "terminated"
 
-    def serve_content(self, path):
-        gateway_basepath = (
-            f"{env.external_protocol}://{env.external_host}/view/{self.key.pathpart}/"
+    def rewrite_text_content(self, cellxgene_content):
+        # for v0.16.0 compatibility, see issue #24
+        gateway_content = (
+            re.sub(
+                '(="|\()/static/',
+                f"\\1{self.gateway_basepath()}static/",
+                cellxgene_content,
+            )
+            .replace("http://fonts.gstatic.com", "https://fonts.gstatic.com")
+            .replace(self.cellxgene_basepath(), self.gateway_basepath())
         )
+        return gateway_content
+
+    def gateway_basepath(self):
+        return f"{env.external_protocol}://{env.external_host}/view/{self.key.pathpart}/"
+
+    def cellxgene_basepath(self):
+        return f"http://127.0.0.1:{self.port}"
+
+    def serve_content(self, path):
+        gateway_basepath = self.gateway_basepath()
         subpath = path[len(self.key.pathpart) :]  # noqa: E203
-            
+
         if len(subpath) == 0:
             r = make_response(f"Redirect to {gateway_basepath}\n", 301)
-            r.headers["location"] = gateway_basepath+querystring()
+            r.headers["location"] = gateway_basepath + querystring()
             return r
         elif self.status == "loading":
             launch_time = datetime.datetime.fromtimestamp(self.launchtime)
             return render_template(
-                "loading.html", launchtime=launch_time, all_output=self.all_output
+                "loading.html",
+                launchtime=launch_time,
+                all_output=self.all_output,
             )
 
-        port = self.port
-        cellxgene_basepath = f"http://127.0.0.1:{port}"
         headers = {}
         copy_headers = [
-            'accept',
-            'accept-encoding',
-            'accept-language',
-            'cache-control',
-            'connection',
-            'content-length',
-            'content-type',
-            'cookie',
-            'host',
-            'origin',
-            'pragma',
-            'referer',
-            'sec-fetch-mode',
-            'sec-fetch-site',
-            'user-agent'
+            "accept",
+            "accept-encoding",
+            "accept-language",
+            "cache-control",
+            "connection",
+            "content-length",
+            "content-type",
+            "cookie",
+            "host",
+            "origin",
+            "pragma",
+            "referer",
+            "sec-fetch-mode",
+            "sec-fetch-site",
+            "user-agent",
         ]
         for h in copy_headers:
             if h in request.headers:
                 headers[h] = request.headers[h]
 
-        full_path = cellxgene_basepath + subpath + querystring()
+        full_path = self.cellxgene_basepath() + subpath + querystring()
 
         if request.method in ["GET", "HEAD", "OPTIONS"]:
-            cellxgene_response = get(
-                full_path, headers=headers
-            )
+            cellxgene_response = get(full_path, headers=headers)
         elif request.method == "PUT":
             cellxgene_response = put(
-                full_path,
-                headers=headers,
-                data=request.data,
+                full_path, headers=headers, data=request.data,
             )
         elif request.method == "POST":
             cellxgene_response = post(
-                full_path,
-                headers=headers,
-                data=request.data,
+                full_path, headers=headers, data=request.data,
             )
         else:
             raise CellxgeneException(
@@ -156,10 +173,9 @@ class CacheEntry:
             )
         content_type = cellxgene_response.headers["content-type"]
         if "text" in content_type:
-            cellxgene_content = cellxgene_response.content.decode()
-            gateway_content = cellxgene_content.replace(
-                "http://fonts.gstatic.com", "https://fonts.gstatic.com"
-            ).replace(cellxgene_basepath, gateway_basepath)
+            gateway_content = self.rewrite_text_content(
+                cellxgene_response.content.decode()
+            )
         else:
             gateway_content = cellxgene_response.content
 
@@ -169,9 +185,7 @@ class CacheEntry:
                 resp_headers[h] = cellxgene_response.headers[h]
 
         gateway_response = make_response(
-            gateway_content,
-            cellxgene_response.status_code,
-            resp_headers,
+            gateway_content, cellxgene_response.status_code, resp_headers,
         )
 
         return gateway_response

cellxgene_gateway/cache_key.py

@@ -7,13 +7,6 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
-import os
-
-from flask_api import status
-
-from cellxgene_gateway import env
-from cellxgene_gateway.cellxgene_exception import CellxgeneException
-
 # There are three kinds of CacheKey:
 # 1) somedir/dataset.h5ad: a dataset
 #    in this case, pathpart == dataset == 'somedir/dataset.h5ad'
@@ -22,6 +15,7 @@ from cellxgene_gateway.cellxgene_exception import CellxgeneException
 # 3) somedir/dataset_annotations: an annotation directory. The corresponding h5ad must exist, but the directory may not.
 #    in this case, pathpart == 'dataset_annotations', dataset == 'somedir/dataset.h5ad'
 
+
 class CacheKey:
     def __init__(self, pathpart, dataset, annotation_file):
         self.pathpart = pathpart

cellxgene_gateway/dir_util.py

@@ -51,8 +51,13 @@ def create_dir(parent_path, dir_name):
     else:
         os.mkdir(full_path)
 
-annotations_suffix = '_annotations'
+
+annotations_suffix = "_annotations"
+
+
 def make_h5ad(el):
-    return el[:-len(annotations_suffix)]+'.h5ad'
+    return el[: -len(annotations_suffix)] + ".h5ad"
+
+
 def make_annotations(el):
-    return el[:-5]+annotations_suffix
+    return el[:-5] + annotations_suffix

cellxgene_gateway/env.py

@@ -13,31 +13,48 @@ import socket
 
 cellxgene_location = os.environ.get("CELLXGENE_LOCATION")
 cellxgene_data = os.environ.get("CELLXGENE_DATA")
+cellxgene_args = os.environ.get("CELLXGENE_ARGS", None)
 gateway_port = int(os.environ.get("GATEWAY_PORT", "5005"))
-external_host = os.environ.get("EXTERNAL_HOST", os.environ.get("GATEWAY_HOST", f"localhost:{gateway_port}"))
-external_protocol = os.environ.get("EXTERNAL_PROTOCOL", os.environ.get("GATEWAY_PROTOCOL", "http"))
-ip = os.environ.get("GATEWAY_IP")
+external_host = os.environ.get(
+    "EXTERNAL_HOST",
+    os.environ.get("GATEWAY_HOST", f"localhost:{gateway_port}"),
+)
+external_protocol = os.environ.get(
+    "EXTERNAL_PROTOCOL", os.environ.get("GATEWAY_PROTOCOL", "http")
+)
+ip = os.environ.get("GATEWAY_IP", "127.0.0.1")
 extra_scripts = os.environ.get("GATEWAY_EXTRA_SCRIPTS")
 ttl = os.environ.get("GATEWAY_TTL")
-enable_upload = os.environ.get("GATEWAY_ENABLE_UPLOAD", "").lower() in ['true', '1']
-enable_annotations = os.environ.get("GATEWAY_ENABLE_ANNOTATIONS", "").lower() in ['true', '1']
+enable_upload = os.environ.get("GATEWAY_ENABLE_UPLOAD", "").lower() in [
+    "true",
+    "1",
+]
+enable_annotations = os.environ.get(
+    "GATEWAY_ENABLE_ANNOTATIONS", ""
+).lower() in ["true", "1"]
+enable_backed_mode = os.environ.get(
+    "GATEWAY_ENABLE_BACKED_MODE", ""
+).lower() in ["true", "1"]
 
 env_vars = {
     "CELLXGENE_LOCATION": cellxgene_location,
     "CELLXGENE_DATA": cellxgene_data,
-    "GATEWAY_IP": ip,
 }
 
 optional_env_vars = {
     "EXTERNAL_HOST": external_host,
     "EXTERNAL_PROTOCOL": external_protocol,
+    "GATEWAY_IP": ip,
     "GATEWAY_PORT": gateway_port,
     "GATEWAY_EXTRA_SCRIPTS": extra_scripts,
     "GATEWAY_TTL": ttl,
     "GATEWAY_ENABLE_UPLOAD": enable_upload,
     "GATEWAY_ENABLE_ANNOTATIONS": enable_annotations,
+    "GATEWAY_ENABLE_BACKED_MODE": enable_backed_mode,
+    "CELLXGENE_ARGS": cellxgene_args,
 }
 
+
 def validate():
     if not all(env_vars.values()):
         raise ValueError(
@@ -52,9 +69,12 @@ def validate():
 
         export CELLXGENE_LOCATION=~/anaconda/envs/cellxgene-dev/bin/cellxgene
         export CELLXGENE_DATA=../cellxgene_data
-        export GATEWAY_IP=127.0.0.1
     """
         )
     else:
-        logging.getLogger("cellxgene_gateway").info(f"Got required env: {env_vars}", )
-        logging.getLogger("cellxgene_gateway").info(f"Got optional env: {optional_env_vars}")
+        logging.getLogger("cellxgene_gateway").info(
+            f"Got required env: {env_vars}",
+        )
+        logging.getLogger("cellxgene_gateway").info(
+            f"Got optional env: {optional_env_vars}"
+        )

cellxgene_gateway/filecrawl.py

@@ -9,7 +9,12 @@
 
 import os
 from cellxgene_gateway import env
-from cellxgene_gateway.dir_util import make_h5ad, make_annotations, annotations_suffix
+from cellxgene_gateway.dir_util import (
+    make_h5ad,
+    make_annotations,
+    annotations_suffix,
+)
+
 
 def recurse_dir(path):
     if not os.path.exists(path):
@@ -17,22 +22,43 @@ def recurse_dir(path):
             "The given path does not exist.", status.HTTP_400_BAD_REQUEST
         )
 
-    all_entries = os.listdir(path)
+    all_entries = sorted(os.listdir(path))
+
     def is_h5ad(el):
-        return el.endswith('.h5ad') and os.path.isfile(os.path.join(path, el))
+        return el.endswith(".h5ad") and os.path.isfile(os.path.join(path, el))
+
     h5ad_entries = [x for x in all_entries if is_h5ad(x)]
-    annotation_dir_entries = [x for x in all_entries if x.endswith(annotations_suffix) and make_h5ad(x) in h5ad_entries]
+    annotation_dir_entries = [
+        x
+        for x in all_entries
+        if x.endswith(annotations_suffix) and make_h5ad(x) in h5ad_entries
+    ]
+
     def list_annotations(el):
         full_path = os.path.join(path, el)
         if not os.path.isdir(full_path):
             entries = []
         else:
-            entries = [{
-                "name": x[:-13] if (len(x) > 13 and x[-13] in ['-','_']) else (
-                    x[:-4] if x.endswith('.csv') else x),
-                "path": os.path.join(full_path, x).replace(env.cellxgene_data, ""),
-            } for x in os.listdir(full_path) if x.endswith('.csv') and os.path.isfile(os.path.join(full_path, x))]
-        return [{"name":'new', "class":'new', "path":full_path.replace(env.cellxgene_data, "")}] + entries
+            entries = [
+                {
+                    "name": x[:-13]
+                    if (len(x) > 13 and x[-13] in ["-", "_"])
+                    else (x[:-4] if x.endswith(".csv") else x),
+                    "path": os.path.join(full_path, x).replace(
+                        env.cellxgene_data, ""
+                    ),
+                }
+                for x in sorted(os.listdir(full_path))
+                if x.endswith(".csv")
+                and os.path.isfile(os.path.join(full_path, x))
+            ]
+        return [
+            {
+                "name": "new",
+                "class": "new",
+                "path": full_path.replace(env.cellxgene_data, ""),
+            }
+        ] + entries
 
     def make_entry(el):
         full_path = os.path.join(path, el)
@@ -57,22 +83,32 @@ def recurse_dir(path):
                 "type": "neither",
             }
 
-    return [make_entry(x) for x in os.listdir(path)]
+    return [make_entry(x) for x in all_entries]
 
 
 def render_entries(entries):
     return "<ul>" + "\n".join([render_entry(e) for e in entries]) + "</ul>"
 
+
 def get_url(entry):
     return f"/view/{ entry['path'].lstrip('/') }"
+
+
 def get_class(entry):
-    return f" class='{entry['class']}'" if 'class' in entry else ''
+    return f" class='{entry['class']}'" if "class" in entry else ""
+
 
 def render_annotations(entry):
-    if len(entry['annotations']) > 0:
-        return ' | annotations: ' + ", ".join([f"<a href='{get_url(a)}'{get_class(a)}>{a['name']}</a>" for a in entry['annotations']])
+    if len(entry["annotations"]) > 0:
+        return " | annotations: " + ", ".join(
+            [
+                f"<a href='{get_url(a)}'{get_class(a)}>{a['name']}</a>"
+                for a in entry["annotations"]
+            ]
+        )
     else:
-        return ''
+        return ""
+
 
 def render_entry(entry):
     if entry["type"] == "file":

cellxgene_gateway/flask_util.py

@@ -9,6 +9,7 @@
 
 from flask import request
 
+
 def querystring():
     qs = request.query_string.decode()
-    return f'?{qs}' if len(qs) > 0 else ''
+    return f"?{qs}" if len(qs) > 0 else ""

cellxgene_gateway/gateway.py

@@ -38,11 +38,15 @@ from cellxgene_gateway.path_util import get_key
 
 app = Flask(__name__)
 
+
 def _force_https(app):
     def wrapper(environ, start_response):
-        environ['wsgi.url_scheme'] = env.external_protocol
+        environ["wsgi.url_scheme"] = env.external_protocol
         return app(environ, start_response)
+
     return wrapper
+
+
 app.wsgi_app = _force_https(app.wsgi_app)
 
 cache = BackendCache()
@@ -114,6 +118,7 @@ def index():
         enable_upload=env.enable_upload,
     )
 
+
 def make_user():
     dir_name = request.form["directory"]
 
@@ -135,13 +140,17 @@ def upload_file():
     upload_dir = request.form["path"]
 
     full_upload_path = os.path.join(env.cellxgene_data, upload_dir)
-    if is_subdir(full_upload_path, env.cellxgene_data) and os.path.isdir(full_upload_path):
+    if is_subdir(full_upload_path, env.cellxgene_data) and os.path.isdir(
+        full_upload_path
+    ):
         if request.method == "POST":
             if "file" in request.files:
                 f = request.files["file"]
                 if f and f.filename.endswith(".h5ad"):
                     f.save(
-                        os.path.join(full_upload_path, secure_filename(f.filename))
+                        os.path.join(
+                            full_upload_path, secure_filename(f.filename)
+                        )
                     )
                     return redirect("/filecrawl.html", code=302)
                 else:
@@ -163,31 +172,44 @@ def upload_file():
 
 
 if env.enable_upload:
-    app.add_url_rule('/make_user', 'make_user', make_user, methods=["POST"])
-    app.add_url_rule('/make_subdir', 'make_subdir', make_subdir, methods=["POST"])
-    app.add_url_rule('/upload_file', 'upload_file', upload_file, methods=["POST"])
+    app.add_url_rule("/make_user", "make_user", make_user, methods=["POST"])
+    app.add_url_rule(
+        "/make_subdir", "make_subdir", make_subdir, methods=["POST"]
+    )
+    app.add_url_rule(
+        "/upload_file", "upload_file", upload_file, methods=["POST"]
+    )
+
+
+def set_no_cache(resp):
+    resp.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
+    resp.headers["Pragma"] = "no-cache"
+    resp.headers["Expires"] = "0"
+    resp.headers["Cache-Control"] = "public, max-age=0"
+    return resp
+
 
 @app.route("/filecrawl.html")
 def filecrawl():
     entries = recurse_dir(env.cellxgene_data)
     rendered_html = render_entries(entries)
-    resp = make_response(render_template(
-        "filecrawl.html",
-        extra_scripts=get_extra_scripts(),
-        rendered_html=rendered_html,
-    ))
-    resp.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
-    resp.headers["Pragma"] = "no-cache"
-    resp.headers["Expires"] = "0"
-    resp.headers['Cache-Control'] = 'public, max-age=0'
-    return resp
+    resp = make_response(
+        render_template(
+            "filecrawl.html",
+            extra_scripts=get_extra_scripts(),
+            rendered_html=rendered_html,
+        )
+    )
+    return set_no_cache(resp)
+
 
 @app.route("/filecrawl/<path:path>")
 def do_filecrawl(path):
     filecrawl_path = os.path.join(env.cellxgene_data, path)
     if not os.path.isdir(filecrawl_path):
         raise CellxgeneException(
-            "Path is not directory: " + filecrawl_path, status.HTTP_400_BAD_REQUEST
+            "Path is not directory: " + filecrawl_path,
+            status.HTTP_400_BAD_REQUEST,
         )
     entries = recurse_dir(filecrawl_path)
     rendered_html = render_entries(entries)
@@ -198,11 +220,16 @@ def do_filecrawl(path):
         path=path,
     )
 
+
 entry_lock = Lock()
+
+
 @app.route("/view/<path:path>", methods=["GET", "PUT", "POST"])
 def do_view(path):
     key = get_key(path)
-    print(f"view path={path}, dataset={key.dataset}, annotation_file= {key.annotation_file}, key={key.pathpart}")
+    print(
+        f"view path={path}, dataset={key.dataset}, annotation_file= {key.annotation_file}, key={key.pathpart}"
+    )
     with entry_lock:
         match = cache.check_entry(key)
         if match is None:
@@ -221,16 +248,25 @@ def do_view(path):
 def do_GET_status():
     return render_template("cache_status.html", entry_list=cache.entry_list)
 
+
 @app.route("/cache_status.json", methods=["GET"])
 def do_GET_status_json():
-    return json.dumps({'launchtime':app.launchtime,
-    'entry_list':[{
-        'dataset': entry.key.dataset,
-        'annotation_file': entry.key.annotation_file,
-        'launchtime': entry.launchtime,
-        'last_access': entry.timestamp,
-        'status': entry.status
-    } for entry in cache.entry_list]})
+    return json.dumps(
+        {
+            "launchtime": app.launchtime,
+            "entry_list": [
+                {
+                    "dataset": entry.key.dataset,
+                    "annotation_file": entry.key.annotation_file,
+                    "launchtime": entry.launchtime,
+                    "last_access": entry.timestamp,
+                    "status": entry.status,
+                }
+                for entry in cache.entry_list
+            ],
+        }
+    )
+
 
 @app.route("/relaunch/<path:path>", methods=["GET"])
 def do_relaunch(path):
@@ -239,7 +275,11 @@ def do_relaunch(path):
     if not match is None:
         match.terminate()
     qs = request.query_string.decode()
-    return redirect(url_for("do_view", path=path) + (f'?{qs}' if len(qs) > 0 else ''), code=302)
+    return redirect(
+        url_for("do_view", path=path) + (f"?{qs}" if len(qs) > 0 else ""),
+        code=302,
+    )
+
 
 @app.route("/terminate/<path:path>", methods=["GET"])
 def do_terminate(path):
@@ -250,8 +290,17 @@ def do_terminate(path):
     return redirect(url_for("do_GET_status"), code=302)
 
 
+@app.route("/metadata/ip_address", methods=["GET"])
+def ip_address():
+    resp = make_response(env.ip)
+    return set_no_cache(resp)
+
+
 def main():
-    logging.basicConfig(level=logging.INFO, format='%(asctime)s:%(name)s:%(levelname)s:%(message)s')
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s:%(name)s:%(levelname)s:%(message)s",
+    )
     env.validate()
     pruner = PruneProcessCache(cache)
 

cellxgene_gateway/path_util.py

@@ -16,6 +16,7 @@ from cellxgene_gateway.cellxgene_exception import CellxgeneException
 from cellxgene_gateway.dir_util import make_h5ad
 from cellxgene_gateway.cache_key import CacheKey
 
+
 def get_key(path):
     if path == "/" or path == "":
         raise CellxgeneException(
@@ -25,10 +26,10 @@ def get_key(path):
     trimmed = path[:-1] if path[-1] == "/" else path
     try:
         # valid paths come in three forms:
-        if trimmed.endswith('.h5ad') and data_file_exists(trimmed):
+        if trimmed.endswith(".h5ad") and data_file_exists(trimmed):
             # 1) somedir/dataset.h5ad: a dataset
             return CacheKey(trimmed, trimmed, None)
-        elif trimmed.endswith('.csv'):
+        elif trimmed.endswith(".csv"):
 
             # 2) somedir/dataset_annotations/saldaal1-T5HMVBNV.csv : an actual annotations file.
             annotations_dir = os.path.split(trimmed)[0]
@@ -36,22 +37,24 @@ def get_key(path):
             if data_file_exists(dataset):
                 data_dir_ensure(annotations_dir)
                 return CacheKey(trimmed, dataset, trimmed)
-        elif trimmed.endswith('_annotations') and data_dir_exists(trimmed):
+        elif trimmed.endswith("_annotations") and data_dir_exists(trimmed):
             # 3) somedir/dataset_annotations: an annotation directory. The corresponding h5ad must exist, but the directory may not.
             dataset = make_h5ad(trimmed)
             if data_file_exists(dataset):
-                return CacheKey(trimmed, dataset, '')
+                return CacheKey(trimmed, dataset, "")
     except CellxgeneException:
         pass
     split = os.path.split(trimmed)
     return get_key(split[0])
 
+
 def validate_exists(file_path):
     if not os.path.exists(file_path):
         raise CellxgeneException(
             "File does not exist: " + file_path, status.HTTP_400_BAD_REQUEST
         )
 
+
 def validate_is_file(file_path):
     validate_exists(file_path)
     if not os.path.isfile(file_path):
@@ -59,6 +62,8 @@ def validate_is_file(file_path):
             "Path is not file: " + file_path, status.HTTP_400_BAD_REQUEST
         )
     return
+
+
 def validate_is_dir(file_path):
     validate_exists(file_path)
     if not os.path.isdir(file_path):
@@ -67,29 +72,36 @@ def validate_is_dir(file_path):
         )
     return
 
+
 def data_file_exists(dataset):
     file_path = os.path.join(env.cellxgene_data, dataset)
     validate_is_file(file_path)
     return True
+
+
 def data_dir_exists(dataset):
     file_path = os.path.join(env.cellxgene_data, dataset)
     validate_is_dir(file_path)
     return True
+
+
 def data_dir_ensure(dataset):
     file_path = os.path.join(env.cellxgene_data, dataset)
     if not os.path.exists(file_path):
         os.makedirs(file_path)
 
+
 def get_file_path(key):
     dataset = key.dataset
     file_path = os.path.join(env.cellxgene_data, dataset)
     validate_is_file(file_path)
     return file_path
 
+
 def get_annotation_file_path(key):
     if key.annotation_file is None:
         return None
-    if key.annotation_file == '':
-        return ''
+    if key.annotation_file == "":
+        return ""
     file_path = os.path.join(env.cellxgene_data, key.annotation_file)
     return file_path

cellxgene_gateway/prune_process_cache.py

@@ -17,7 +17,7 @@ from cellxgene_gateway.env import ttl
 class PruneProcessCache:
     def __init__(self, cache):
         self.cache = cache
-        self.expire_seconds = (3600 if ttl is None else int(ttl))
+        self.expire_seconds = 3600 if ttl is None else int(ttl)
 
     def __call__(self):
         while True:
@@ -27,14 +27,24 @@ class PruneProcessCache:
     def prune(self):
         timestamp = current_time_stamp()
         cutoff = timestamp - self.expire_seconds
-        processes_to_delete = [p for p in self.cache.entry_list if  p.timestamp < cutoff]
-        processes_to_keep = [p for p in self.cache.entry_list if not p.timestamp < cutoff]
+        processes_to_delete = [
+            p for p in self.cache.entry_list if p.timestamp < cutoff
+        ]
+        processes_to_keep = [
+            p for p in self.cache.entry_list if not p.timestamp < cutoff
+        ]
         logger = logging.getLogger("cellxgene_gateway")
-        logger.debug(f"Cutoff {cutoff} = timestamp {timestamp} - expire seconds {self.expire_seconds} , keeping {processes_to_keep}")
-        
+        logger.debug(
+            f"Cutoff {cutoff} = timestamp {timestamp} - expire seconds {self.expire_seconds} , keeping {processes_to_keep}"
+        )
+
         for process in processes_to_delete:
             try:
-                logger.info(f"pruning process {process.pid} ({process.key.dataset})")
+                logger.info(
+                    f"pruning process {process.pid} ({process.key.dataset})"
+                )
                 self.cache.prune(process)
             except Exception:
-                logger.exception("failed to prune process {process.pid} ({process.dataset})")
+                logger.exception(
+                    "failed to prune process {process.pid} ({process.dataset})"
+                )

cellxgene_gateway/subprocess_backend.py

@@ -11,30 +11,43 @@ import logging
 import subprocess
 
 from flask_api import status
-from cellxgene_gateway.env import enable_annotations
+from cellxgene_gateway.env import (
+    enable_annotations,
+    enable_backed_mode,
+    cellxgene_args,
+)
 from cellxgene_gateway.process_exception import ProcessException
 from cellxgene_gateway.dir_util import make_annotations
 from cellxgene_gateway.path_util import get_file_path, get_annotation_file_path
 
+
 class SubprocessBackend:
     def __init__(self):
         pass
 
-    def create_cmd(self, cellxgene_loc, file_path, port, scripts, annotation_file_path):
+    def create_cmd(
+        self, cellxgene_loc, file_path, port, scripts, annotation_file_path
+    ):
         if enable_annotations and not annotation_file_path is None:
-            annotation_args_prefix = " --experimental-annotations"
             if annotation_file_path == "":
-                annotation_args = f"{annotation_args_prefix} --experimental-annotations-output-dir {make_annotations(file_path)}"
+                extra_args = (
+                    f" --annotations-dir {make_annotations(file_path)}"
+                )
             else:
-                annotation_args = f"{annotation_args_prefix} --experimental-annotations-file {annotation_file_path}"
+                extra_args = f" --annotations-file {annotation_file_path}"
         else:
-            annotation_args = ""
+            extra_args = " --disable-annotations"
+        if enable_backed_mode:
+            extra_args += " --backed"
+        if not cellxgene_args is None:
+            extra_args += f" {cellxgene_args}"
+
         cmd = (
             f"yes | {cellxgene_loc} launch {file_path}"
             + " --port "
             + str(port)
             + " --host 127.0.0.1"
-            + annotation_args
+            + extra_args
         )
 
         for s in scripts:
@@ -45,7 +58,11 @@ class SubprocessBackend:
     def launch(self, cellxgene_loc, scripts, cache_entry):
 
         cmd = self.create_cmd(
-            cellxgene_loc, get_file_path(cache_entry.key), cache_entry.port, scripts, get_annotation_file_path(cache_entry.key)
+            cellxgene_loc,
+            get_file_path(cache_entry.key),
+            cache_entry.port,
+            scripts,
+            get_annotation_file_path(cache_entry.key),
         )
         logging.getLogger("cellxgene_gateway").info(f"launching {cmd}")
         process = subprocess.Popen(

environment.yml

@@ -6,6 +6,8 @@ dependencies:
 - requests
 - flask
 - psutil
+- black
+- pip
 - pip:
   - flask-api
-  - cellxgene==0.14.1
+  - cellxgene>=0.15

requirements.txt

@@ -1,4 +1,4 @@
-cellxgene==0.14.1
+cellxgene>=0.15
 flask
 flask_api
 psutil

setup.py

@@ -1,5 +1,26 @@
 import os
-from setuptools import setup
+import codecs
+from setuptools import find_packages, setup
+import sys
+
+if sys.version_info < (3, 6):
+    sys.exit("Sorry, Python < 3.6 is not supported")
+
+
+def read(rel_path):
+    here = os.path.abspath(os.path.dirname(__file__))
+    with codecs.open(os.path.join(here, rel_path), "r") as fp:
+        return fp.read()
+
+
+def get_version(rel_path):
+    for line in read(rel_path).splitlines():
+        if line.startswith("__version__"):
+            delim = '"' if '"' in line else "'"
+            return line.split(delim)[1]
+    else:
+        raise RuntimeError("Unable to find version string.")
+
 
 def parse_requirements():
     reqs = []
@@ -8,6 +29,7 @@ def parse_requirements():
             reqs.append(l.strip("\n"))
     return reqs
 
+
 with open("README.md", "r") as fh:
     long_description = fh.read()
 
@@ -17,7 +39,7 @@ setup(
     # mandatory
     name="cellxgene-gateway",
     # mandatory
-    version="0.1.0",
+    version=get_version("cellxgene_gateway/__init__.py"),
     # mandatory
     author="Niket Patel, Yohann Potier, Alok Saldanha",
     author_email="alok.saldanha@novartis.com",
@@ -32,13 +54,14 @@ setup(
         "cellxgene_gateway": [
             "static/css/homepagestyle.css",
             "static/nibr.ico",
-            "templates/*.html"
-    ]},
-    data_files=[('', ['README.md', 'LICENSE'])],
+            "templates/*.html",
+        ]
+    },
+    data_files=[("", ["README.md", "LICENSE"])],
     install_requires=install_reqs,
     entry_points={
         "console_scripts": ["cellxgene-gateway=cellxgene_gateway.gateway:main"]
     },
     classifiers=["Topic :: Scientific/Engineering :: Visualization"],
-    python_requires='>=3.6',
+    python_requires=">=3.6",
 )

tests/test_cache_entry.py

@@ -0,0 +1,23 @@
+import unittest
+from cellxgene_gateway.cache_entry import CacheEntry
+from cellxgene_gateway.cache_key import CacheKey
+
+key = CacheKey("czi/pbmc3k.h5ad", "pbmc3k.h5ad", "tmp.csv")
+
+
+class TestRenderEntry(unittest.TestCase):
+    def test_GIVEN_absolute_static_url_THEN_include_path(self):
+        actual = CacheEntry.for_key(key, 8000).rewrite_text_content(
+            "src:url(/static/assets/"
+        )
+        expected = (
+            "src:url(http://localhost:5005/view/czi/pbmc3k.h5ad/static/assets/"
+        )
+        self.assertEqual(actual, expected)
+
+    def test_GIVEN_absolute_src_THEN_include_path(self):
+        actual = CacheEntry.for_key(key, 8000).rewrite_text_content(
+            '<link rel="shortcut icon" href="/static/assets/favicon.ico">'
+        )
+        expected = '<link rel="shortcut icon" href="http://localhost:5005/view/czi/pbmc3k.h5ad/static/assets/favicon.ico">'
+        self.assertEqual(actual, expected)

tests/test_dir_util.py

@@ -1,38 +1,49 @@
 import unittest
 from unittest.mock import MagicMock, patch
-from cellxgene_gateway.dir_util import render_entry
+from cellxgene_gateway.filecrawl import render_entry
+
 
 class TestRenderEntry(unittest.TestCase):
     def test_GIVEN_path_both_slash_THEN_view_has_single_slash(self):
         entry = {
-                "path": "/somepath/",
-                "name": "entry",
-                "type": "file",
-            }
+            "path": "/somepath/",
+            "name": "entry",
+            "type": "file",
+            "annotations": [],
+            "children": [],
+        }
         rendered = render_entry(entry)
-        self.assertIn('view/somepath', rendered)
+        self.assertIn("view/somepath", rendered)
+
     def test_GIVEN_path_starts_slash_THEN_view_has_single_slash(self):
         entry = {
-                "path": "/somepath",
-                "name": "entry",
-                "type": "file",
-            }
+            "path": "/somepath",
+            "name": "entry",
+            "type": "file",
+            "annotations": [],
+            "children": [],
+        }
         rendered = render_entry(entry)
-        self.assertIn('view/somepath', rendered)
+        self.assertIn("view/somepath", rendered)
+
     def test_GIVEN_path_ends_slash_THEN_view_has_single_slash(self):
         entry = {
-                "path": "somepath/",
-                "name": "entry",
-                "type": "file",
-            }
+            "path": "somepath/",
+            "name": "entry",
+            "type": "file",
+            "annotations": [],
+            "children": [],
+        }
         rendered = render_entry(entry)
-        self.assertIn('view/somepath', rendered)
+        self.assertIn("view/somepath", rendered)
+
     def test_GIVEN_path_no_slash_THEN_view_has_single_slash(self):
         entry = {
-                "path": "somepath",
-                "name": "entry",
-                "type": "file",
-            }
+            "path": "somepath",
+            "name": "entry",
+            "type": "file",
+            "annotations": [],
+            "children": [],
+        }
         rendered = render_entry(entry)
-        self.assertIn('view/somepath', rendered)
-        
+        self.assertIn("view/somepath", rendered)

tests/test_extra_scripts.py

@@ -2,22 +2,24 @@ import unittest
 from unittest.mock import MagicMock, patch
 from cellxgene_gateway.extra_scripts import get_extra_scripts
 
+
 class TestExtraScripts(unittest.TestCase):
-    @patch('cellxgene_gateway.env.extra_scripts', new='["abc","def"]')
+    @patch("cellxgene_gateway.env.extra_scripts", new='["abc","def"]')
     def test_GIVEN_two_scripts_THEN_returns_two_strings(self):
-        self.assertEqual(get_extra_scripts(), ['abc', 'def'])
+        self.assertEqual(get_extra_scripts(), ["abc", "def"])
 
-    @patch('cellxgene_gateway.env.extra_scripts', new='["abc", "def"]')
+    @patch("cellxgene_gateway.env.extra_scripts", new='["abc", "def"]')
     def test_GIVEN_two_scripts_space_THEN_returns_two_strings(self):
-        self.assertEqual(get_extra_scripts(), ['abc', 'def'])
+        self.assertEqual(get_extra_scripts(), ["abc", "def"])
 
-    @patch('cellxgene_gateway.env.extra_scripts', new=None)
+    @patch("cellxgene_gateway.env.extra_scripts", new=None)
     def test_GIVEN_none_THEN_returns_empty_array(self):
         self.assertEqual(get_extra_scripts(), [])
 
-    @patch('cellxgene_gateway.env.extra_scripts', new='[]')
+    @patch("cellxgene_gateway.env.extra_scripts", new="[]")
     def test_GIVEN_empty_string_THEN_returns_empty_array(self):
         self.assertEqual(get_extra_scripts(), [])
 
-if __name__ == '__main__':
-    unittest.main()
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_prune_process_cache.py

@@ -3,11 +3,12 @@ from unittest.mock import MagicMock, patch
 from cellxgene_gateway.cache_entry import CacheEntry
 from cellxgene_gateway.backend_cache import BackendCache
 
+
 class TestPruneProcessCache(unittest.TestCase):
-    @patch('cellxgene_gateway.util.current_time_stamp', new=lambda:0)
-    @patch('cellxgene_gateway.env.ttl', new='10')
-    @patch('cellxgene_gateway.cache_entry.CacheEntry')
-    @patch('cellxgene_gateway.cache_entry.CacheEntry')
+    @patch("cellxgene_gateway.util.current_time_stamp", new=lambda: 0)
+    @patch("cellxgene_gateway.env.ttl", new="10")
+    @patch("cellxgene_gateway.cache_entry.CacheEntry")
+    @patch("cellxgene_gateway.cache_entry.CacheEntry")
     def test_GIVEN_one_old_one_new_THEN_prune_old(self, old, new):
         from cellxgene_gateway.prune_process_cache import PruneProcessCache
 
@@ -22,5 +23,6 @@ class TestPruneProcessCache(unittest.TestCase):
         self.assertEqual(len(cache.entry_list), 1)
         self.assertEqual(cache.entry_list[0], new)
 
-if __name__ == '__main__':
-    unittest.main()
+
+if __name__ == "__main__":
+    unittest.main()