ignore patch files

Mark version 0.1.0
Add python_requires
add long_description

.gitignore

@@ -136,3 +136,6 @@ dmypy.json
 .pyre/
 
 # End of https://www.gitignore.io/api/python
+
+*.patch
+.vscode

Changelog.md

@@ -0,0 +1,7 @@
+# 0.2.0
+
+Incrementing minor version since the changes for 0.15 are breaking, and we may want to release bugfixes from 0.1.0 branch.
+
+# 0.1.1
+
+Added support for cellxgene 0.15

README.md

@@ -48,9 +48,6 @@ wget https://github.com/chanzuckerberg/cellxgene/raw/master/example-dataset/pbmc
 ```bash
 export CELLXGENE_DATA=../cellxgene_data  # change this directory if you put data in a different place.
 export CELLXGENE_LOCATION=`which cellxgene`
-export GATEWAY_HOST=localhost:5005
-export GATEWAY_PROTOCOL=http
-export GATEWAY_IP=127.0.0.1
 ```
 
 3. Now, execute the cellxgene gateway:
@@ -63,12 +60,15 @@ Here's what the environment variables mean:
 
 * `CELLXGENE_LOCATION` - the location of the cellxgene executable, e.g. `~/anaconda2/envs/cellxgene/bin/cellxgene`
 * `CELLXGENE_DATA` - a directory that can contain subdirectories with `.h5ad` data files, *without* trailing slash, e.g. `/mnt/cellxgene_data`
-* `GATEWAY_HOST` - the hostname and port that the gateway will run on, typically `localhost:5005` if running locally
-* `GATEWAY_PROTOCOL` - typically http when running locally, can be https when deployed if the gateway is behind a load balancer or reverse proxy.
-* `GATEWAY_IP` - ip addess of instance gateway is running on, mostly used to display SSH instructions
 Optional environment variables:
+* `EXTERNAL_HOST` - the hostname and port from the perspective of the web browser, typically `localhost:5005` if running locally. Defaults to "localhost:{GATEWAY_PORT}"
+* `EXTERNAL_PROTOCOL` - typically http when running locally, can be https when deployed if the gateway is behind a load balancer or reverse proxy that performs https termination. Default value "http"
+* `GATEWAY_IP` - ip addess of instance gateway is running on, mostly used to display SSH instructions. Defaults to `socket.gethostbyname(socket.gethostname())`
+* `GATEWAY_PORT` - local port that the gateway should bind to, defaults to 5005
 * `GATEWAY_EXTRA_SCRIPTS` - JSON array of script paths, will be embedded into each page and forwarded with `--scripts` to cellxgene server
 * `GATEWAY_ENABLE_UPLOAD` - Set to `true` or `1` to enable HTTP uploads. This is not recommended for a public server.
+* `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations. 
+* `GATEWAY_ENABLE_BACKED_MODE` - Set to `true` or to `1` to load AnnData in file-backed mode. This saves memory and speeds up launch time but may reduce overall performance.  
 
 The defaults should be fine if you set up a venv and cellxgene_data folder as above.
 

cellxgene_gateway/__init__.py

@@ -6,3 +6,5 @@
 # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
+
+__version__ = "0.2.0"

cellxgene_gateway/backend_cache.py

@@ -33,12 +33,12 @@ class BackendCache:
         contents = self.entry_list
         return [c.port for c in contents]
 
-    def check_entry(self, dataset):
+    def check_entry(self, key):
         contents = self.entry_list
         matches = [
             c
             for c in contents
-            if c.dataset == dataset and c.status != "terminated"
+            if c.key.dataset == key.dataset and c.key.annotation_file == key.annotation_file and c.status != "terminated"
         ]
 
         if len(matches) == 0:
@@ -51,13 +51,14 @@ class BackendCache:
                 "Found " + str(len(matches)) + " for " + dataset,
             )
 
-    def create_entry(self, dataset, file_path, scripts):
+    def create_entry(self, key, scripts):
         port = 8000
         existing_ports = self.get_ports()
+
         while (port in existing_ports) or is_port_in_use(port):
             port += 1
 
-        entry = CacheEntry.for_dataset(dataset, file_path, port)
+        entry = CacheEntry.for_key(key, port)
 
         background_thread = Thread(
             target=process_backend.launch,

cellxgene_gateway/cache_entry.py

@@ -8,20 +8,21 @@
 # the specific language governing permissions and limitations under the License.
 import psutil
 import logging
+import datetime
 
-from flask import make_response, request
+from flask import make_response, request, render_template
 from requests import get, post, put
 
 from cellxgene_gateway import env
 from cellxgene_gateway.cellxgene_exception import CellxgeneException
 from cellxgene_gateway.util import current_time_stamp
+from cellxgene_gateway.flask_util import querystring
 
 class CacheEntry:
     def __init__(
         self,
         pid,
-        dataset,
-        file_path,
+        key,
         port,
         launchtime,
         timestamp,
@@ -32,8 +33,7 @@ class CacheEntry:
         http_status,
     ):
         self.pid = pid
-        self.dataset = dataset
-        self.file_path = file_path
+        self.key = key
         self.port = port
         self.launchtime = launchtime
         self.timestamp = timestamp
@@ -44,11 +44,11 @@ class CacheEntry:
         self.http_status = http_status
 
     @classmethod
-    def for_dataset(cls, dataset, file_path, port):
+    def for_key(cls, key, port):
+
         return cls(
             None,
-            dataset,
-            file_path,
+            key,
             port,
             current_time_stamp(),
             current_time_stamp(),
@@ -93,45 +93,62 @@ class CacheEntry:
         self.status = "terminated"
 
     def serve_content(self, path):
-        dataset = self.dataset
-
         gateway_basepath = (
-            f"{env.gateway_protocol}://{env.gateway_host}/view/{dataset}/"
+            f"{env.external_protocol}://{env.external_host}/view/{self.key.pathpart}/"
         )
-        subpath = path[len(dataset) :]  # noqa: E203
-
+        subpath = path[len(self.key.pathpart) :]  # noqa: E203
+            
         if len(subpath) == 0:
             r = make_response(f"Redirect to {gateway_basepath}\n", 301)
-            r.headers["location"] = gateway_basepath
+            r.headers["location"] = gateway_basepath+querystring()
             return r
+        elif self.status == "loading":
+            launch_time = datetime.datetime.fromtimestamp(self.launchtime)
+            return render_template(
+                "loading.html", launchtime=launch_time, all_output=self.all_output
+            )
 
         port = self.port
         cellxgene_basepath = f"http://127.0.0.1:{port}"
-
         headers = {}
+        copy_headers = [
+            'accept',
+            'accept-encoding',
+            'accept-language',
+            'cache-control',
+            'connection',
+            'content-length',
+            'content-type',
+            'cookie',
+            'host',
+            'origin',
+            'pragma',
+            'referer',
+            'sec-fetch-mode',
+            'sec-fetch-site',
+            'user-agent'
+        ]
+        for h in copy_headers:
+            if h in request.headers:
+                headers[h] = request.headers[h]
 
-        if "accept" in request.headers:
-            headers["accept"] = request.headers["accept"]
-        if "user-agent" in request.headers:
-            headers["user-agent"] = request.headers["user-agent"]
-        if "content-type" in request.headers:
-            headers["content-type"] = request.headers["content-type"]
+        full_path = cellxgene_basepath + subpath + querystring()
 
         if request.method in ["GET", "HEAD", "OPTIONS"]:
             cellxgene_response = get(
-                cellxgene_basepath + subpath, headers=headers
+                full_path, headers=headers
             )
         elif request.method == "PUT":
             cellxgene_response = put(
-                cellxgene_basepath + subpath,
+                full_path,
                 headers=headers,
-                data=request.data.decode(),
+                data=request.data,
             )
         elif request.method == "POST":
             cellxgene_response = post(
-                cellxgene_basepath + subpath,
+                full_path,
                 headers=headers,
-                data=request.data.decode(),
+                data=request.data,
             )
         else:
             raise CellxgeneException(
@@ -146,10 +163,15 @@ class CacheEntry:
         else:
             gateway_content = cellxgene_response.content
 
+        resp_headers = {}
+        for h in copy_headers:
+            if h in cellxgene_response.headers:
+                resp_headers[h] = cellxgene_response.headers[h]
+
         gateway_response = make_response(
             gateway_content,
             cellxgene_response.status_code,
-            {"Content-Type": content_type},
+            resp_headers,
         )
 
         return gateway_response

cellxgene_gateway/cache_key.py

@@ -0,0 +1,29 @@
+# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
+# under the Apache License, Version 2.0 (the "License"); you may not use
+# this file except in compliance with the License. You may obtain a copy
+# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
+# required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+# the specific language governing permissions and limitations under the License.
+
+import os
+
+from flask_api import status
+
+from cellxgene_gateway import env
+from cellxgene_gateway.cellxgene_exception import CellxgeneException
+
+# There are three kinds of CacheKey:
+# 1) somedir/dataset.h5ad: a dataset
+#    in this case, pathpart == dataset == 'somedir/dataset.h5ad'
+# 2) somedir/dataset_annotations/saldaal1-T5HMVBNV.csv : an actual annotaitons file.
+#    in this case, pathpart == 'dataset_annotations/saldaal1-T5HMVBNV.csv', dataset == 'somedir/dataset.h5ad'
+# 3) somedir/dataset_annotations: an annotation directory. The corresponding h5ad must exist, but the directory may not.
+#    in this case, pathpart == 'dataset_annotations', dataset == 'somedir/dataset.h5ad'
+
+class CacheKey:
+    def __init__(self, pathpart, dataset, annotation_file):
+        self.pathpart = pathpart
+        self.dataset = dataset
+        self.annotation_file = annotation_file

cellxgene_gateway/dir_util.py

@@ -51,44 +51,8 @@ def create_dir(parent_path, dir_name):
     else:
         os.mkdir(full_path)
 
-
-def recurse_dir(path):
-    if not os.path.exists(path):
-        raise CellxgeneException(
-            "The given path does not exist.", status.HTTP_400_BAD_REQUEST
-        )
-
-    def make_entry(el):
-        full_path = os.path.join(path, el)
-        if os.path.isfile(full_path):
-            return {
-                "path": full_path.replace(env.cellxgene_data, ""),
-                "name": el,
-                "type": "file",
-            }
-        elif os.path.isdir(full_path):
-            return {
-                "path": full_path,
-                "name": el,
-                "type": "directory",
-                "children": recurse_dir(full_path),
-            }
-        else:
-            raise CellxgeneException(
-                "Given path is neither file nor directory.",
-                status.HTTP_400_BAD_REQUEST,
-            )
-
-    return [make_entry(x) for x in os.listdir(path)]
-
-
-def render_entries(entries):
-    return "<ul>" + "\n".join([render_entry(e) for e in entries]) + "</ul>"
-
-
-def render_entry(entry):
-    if entry["type"] == "file":
-        url = 'view' + '/' + entry['path'].lstrip("/")
-        return f"<li> <a href='{ url}'>{entry['name']}</a></li>"
-    elif entry["type"] == "directory":
-        return f"<li>{entry['name']}{render_entries(entry['children'])}</li>"
+annotations_suffix = '_annotations'
+def make_h5ad(el):
+    return el[:-len(annotations_suffix)]+'.h5ad'
+def make_annotations(el):
+    return el[:-5]+annotations_suffix

cellxgene_gateway/env.py

@@ -9,28 +9,35 @@
 
 import os
 import logging
+import socket
 
 cellxgene_location = os.environ.get("CELLXGENE_LOCATION")
 cellxgene_data = os.environ.get("CELLXGENE_DATA")
-gateway_host = os.environ.get("GATEWAY_HOST")
-gateway_protocol = os.environ.get("GATEWAY_PROTOCOL")
+gateway_port = int(os.environ.get("GATEWAY_PORT", "5005"))
+external_host = os.environ.get("EXTERNAL_HOST", os.environ.get("GATEWAY_HOST", f"localhost:{gateway_port}"))
+external_protocol = os.environ.get("EXTERNAL_PROTOCOL", os.environ.get("GATEWAY_PROTOCOL", "http"))
 ip = os.environ.get("GATEWAY_IP")
 extra_scripts = os.environ.get("GATEWAY_EXTRA_SCRIPTS")
 ttl = os.environ.get("GATEWAY_TTL")
 enable_upload = os.environ.get("GATEWAY_ENABLE_UPLOAD", "").lower() in ['true', '1']
+enable_annotations = os.environ.get("GATEWAY_ENABLE_ANNOTATIONS", "").lower() in ['true', '1']
+enable_backed_mode = os.environ.get("GATEWAY_ENABLE_BACKED_MODE", "").lower() in ['true', '1']
 
 env_vars = {
     "CELLXGENE_LOCATION": cellxgene_location,
     "CELLXGENE_DATA": cellxgene_data,
-    "GATEWAY_HOST": gateway_host,
-    "GATEWAY_PROTOCOL": gateway_protocol,
     "GATEWAY_IP": ip,
 }
 
 optional_env_vars = {
+    "EXTERNAL_HOST": external_host,
+    "EXTERNAL_PROTOCOL": external_protocol,
+    "GATEWAY_PORT": gateway_port,
     "GATEWAY_EXTRA_SCRIPTS": extra_scripts,
     "GATEWAY_TTL": ttl,
     "GATEWAY_ENABLE_UPLOAD": enable_upload,
+    "GATEWAY_ENABLE_ANNOTATIONS": enable_annotations,
+    "GATEWAY_ENABLE_BACKED_MODE": enable_backed_mode,
 }
 
 def validate():
@@ -47,8 +54,6 @@ def validate():
 
         export CELLXGENE_LOCATION=~/anaconda/envs/cellxgene-dev/bin/cellxgene
         export CELLXGENE_DATA=../cellxgene_data
-        export GATEWAY_HOST=localhost:5005
-        export GATEWAY_PROTOCOL=http
         export GATEWAY_IP=127.0.0.1
     """
         )

cellxgene_gateway/extra_scripts.py

@@ -14,6 +14,6 @@ from json import loads
 def get_extra_scripts():
     # can be array of script tags to inject on every page, e.g. for google analytics could be
     # ['https://www.googletagmanager.com/gtag/js?id=UA-123456-2',
-    #  f"{env.gateway_protocol}://{env.gateway_host}/static/js/google_ua.js"]
+    #  f"{env.external_protocol}://{env.external_host}/static/js/google_ua.js"]
     # where google_ua.js is a script you add to the static/js folder prior to deployment.
     return [] if env.extra_scripts is None else loads(env.extra_scripts)

cellxgene_gateway/filecrawl.py

@@ -0,0 +1,84 @@
+# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
+# under the Apache License, Version 2.0 (the "License"); you may not use
+# this file except in compliance with the License. You may obtain a copy
+# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
+# required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+# the specific language governing permissions and limitations under the License.
+
+import os
+from cellxgene_gateway import env
+from cellxgene_gateway.dir_util import make_h5ad, make_annotations, annotations_suffix
+
+def recurse_dir(path):
+    if not os.path.exists(path):
+        raise CellxgeneException(
+            "The given path does not exist.", status.HTTP_400_BAD_REQUEST
+        )
+
+    all_entries = sorted(os.listdir(path))
+    def is_h5ad(el):
+        return el.endswith('.h5ad') and os.path.isfile(os.path.join(path, el))
+    h5ad_entries = [x for x in all_entries if is_h5ad(x)]
+    annotation_dir_entries = [x for x in all_entries if x.endswith(annotations_suffix) and make_h5ad(x) in h5ad_entries]
+    def list_annotations(el):
+        full_path = os.path.join(path, el)
+        if not os.path.isdir(full_path):
+            entries = []
+        else:
+            entries = [{
+                "name": x[:-13] if (len(x) > 13 and x[-13] in ['-','_']) else (
+                    x[:-4] if x.endswith('.csv') else x),
+                "path": os.path.join(full_path, x).replace(env.cellxgene_data, ""),
+            } for x in sorted(os.listdir(full_path)) if x.endswith('.csv') and os.path.isfile(os.path.join(full_path, x))]
+        return [{"name":'new', "class":'new', "path":full_path.replace(env.cellxgene_data, "")}] + entries
+
+    def make_entry(el):
+        full_path = os.path.join(path, el)
+        if el in h5ad_entries:
+            return {
+                "path": full_path.replace(env.cellxgene_data, ""),
+                "name": el,
+                "type": "file",
+                "annotations": list_annotations(make_annotations(el)),
+            }
+        elif os.path.isdir(full_path) and el not in annotation_dir_entries:
+            return {
+                "path": full_path.replace(env.cellxgene_data, ""),
+                "name": el,
+                "type": "directory",
+                "children": recurse_dir(full_path),
+            }
+        else:
+            return {
+                "path": full_path,
+                "name": el,
+                "type": "neither",
+            }
+
+    return [make_entry(x) for x in all_entries]
+
+
+def render_entries(entries):
+    return "<ul>" + "\n".join([render_entry(e) for e in entries]) + "</ul>"
+
+def get_url(entry):
+    return f"/view/{ entry['path'].lstrip('/') }"
+def get_class(entry):
+    return f" class='{entry['class']}'" if 'class' in entry else ''
+
+def render_annotations(entry):
+    if len(entry['annotations']) > 0:
+        return ' | annotations: ' + ", ".join([f"<a href='{get_url(a)}'{get_class(a)}>{a['name']}</a>" for a in entry['annotations']])
+    else:
+        return ''
+
+def render_entry(entry):
+    if entry["type"] == "file":
+        return f"<li> <a href='{ get_url(entry) }'>{entry['name']}</a> {render_annotations(entry)}</li>"
+    elif entry["type"] == "directory":
+        url = f"/filecrawl/{entry['path'].lstrip('/')}"
+        return f"<li><a href='{url}'>{entry['name']}</a>{render_entries(entry['children'])}</li>"
+    else:
+        return ""

cellxgene_gateway/flask_util.py

@@ -0,0 +1,14 @@
+# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
+# under the Apache License, Version 2.0 (the "License"); you may not use
+# this file except in compliance with the License. You may obtain a copy
+# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
+# required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+# the specific language governing permissions and limitations under the License.
+
+from flask import request
+
+def querystring():
+    qs = request.query_string.decode()
+    return f'?{qs}' if len(qs) > 0 else ''

cellxgene_gateway/gateway.py

@@ -8,7 +8,6 @@
 # the specific language governing permissions and limitations under the License.
 
 # import BaseHTTPServer
-import datetime
 import os
 import logging
 from threading import Thread, Lock
@@ -17,27 +16,37 @@ import json
 from flask import (
     Flask,
     redirect,
+    make_response,
     render_template,
     request,
     send_from_directory,
     url_for,
 )
 from flask_api import status
-from werkzeug import secure_filename
+from werkzeug.utils import secure_filename
 
 from cellxgene_gateway import env
 from cellxgene_gateway.backend_cache import BackendCache
 from cellxgene_gateway.cellxgene_exception import CellxgeneException
-from cellxgene_gateway.dir_util import create_dir, recurse_dir, render_entries, is_subdir
+from cellxgene_gateway.dir_util import create_dir, is_subdir
+from cellxgene_gateway.filecrawl import recurse_dir, render_entries
 from cellxgene_gateway.extra_scripts import get_extra_scripts
-from cellxgene_gateway.path_util import get_dataset, get_file_path
 from cellxgene_gateway.process_exception import ProcessException
 from cellxgene_gateway.prune_process_cache import PruneProcessCache
 from cellxgene_gateway.util import current_time_stamp
+from cellxgene_gateway.path_util import get_key
 
 app = Flask(__name__)
+
+def _force_https(app):
+    def wrapper(environ, start_response):
+        environ['wsgi.url_scheme'] = env.external_protocol
+        return app(environ, start_response)
+    return wrapper
+app.wsgi_app = _force_https(app.wsgi_app)
+
 cache = BackendCache()
-location = f"{env.gateway_protocol}://{env.gateway_host}"
+location = f"{env.external_protocol}://{env.external_host}"
 
 
 @app.errorhandler(CellxgeneException)
@@ -73,7 +82,8 @@ def handle_invalid_process(error):
             http_status=error.http_status,
             stdout=error.stdout,
             stderr=error.stderr,
-            dataset=error.dataset,
+            dataset=error.key.dataset,
+            annotation_file=error.key.annotation_file,
         ),
         error.http_status,
     )
@@ -149,7 +159,7 @@ def upload_file():
             "Invalid directory.", status.HTTP_400_BAD_REQUEST
         )
 
-    return redirect(env.location, code=302)
+    return redirect(location, code=302)
 
 
 if env.enable_upload:
@@ -159,35 +169,50 @@ if env.enable_upload:
 
 @app.route("/filecrawl.html")
 def filecrawl():
-
     entries = recurse_dir(env.cellxgene_data)
     rendered_html = render_entries(entries)
+    resp = make_response(render_template(
+        "filecrawl.html",
+        extra_scripts=get_extra_scripts(),
+        rendered_html=rendered_html,
+    ))
+    resp.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
+    resp.headers["Pragma"] = "no-cache"
+    resp.headers["Expires"] = "0"
+    resp.headers['Cache-Control'] = 'public, max-age=0'
+    return resp
+
+@app.route("/filecrawl/<path:path>")
+def do_filecrawl(path):
+    filecrawl_path = os.path.join(env.cellxgene_data, path)
+    if not os.path.isdir(filecrawl_path):
+        raise CellxgeneException(
+            "Path is not directory: " + filecrawl_path, status.HTTP_400_BAD_REQUEST
+        )
+    entries = recurse_dir(filecrawl_path)
+    rendered_html = render_entries(entries)
     return render_template(
         "filecrawl.html",
         extra_scripts=get_extra_scripts(),
         rendered_html=rendered_html,
+        path=path,
     )
 
 entry_lock = Lock()
 @app.route("/view/<path:path>", methods=["GET", "PUT", "POST"])
 def do_view(path):
-    dataset = get_dataset(path)
-    file_path = get_file_path(dataset)
+    key = get_key(path)
+    print(f"view path={path}, dataset={key.dataset}, annotation_file= {key.annotation_file}, key={key.pathpart}")
     with entry_lock:
-        match = cache.check_entry(dataset)
+        match = cache.check_entry(key)
         if match is None:
             uascripts = get_extra_scripts()
-            match = cache.create_entry(dataset, file_path, uascripts)
+            match = cache.create_entry(key, uascripts)
 
     match.timestamp = current_time_stamp()
 
-    if match.status == "loaded":
+    if match.status == "loaded" or match.status == "loading":
         return match.serve_content(path)
-    elif match.status == "loading":
-        launch_time = datetime.datetime.fromtimestamp(match.launchtime)
-        return render_template(
-            "loading.html", launchtime=launch_time, all_output=match.all_output
-        )
     elif match.status == "error":
         raise ProcessException.from_cache_entry(match)
 
@@ -200,7 +225,8 @@ def do_GET_status():
 def do_GET_status_json():
     return json.dumps({'launchtime':app.launchtime,
     'entry_list':[{
-        'dataset': entry.dataset,
+        'dataset': entry.key.dataset,
+        'annotation_file': entry.key.annotation_file,
         'launchtime': entry.launchtime,
         'last_access': entry.timestamp,
         'status': entry.status
@@ -208,16 +234,17 @@ def do_GET_status_json():
 
 @app.route("/relaunch/<path:path>", methods=["GET"])
 def do_relaunch(path):
-    dataset = get_dataset(path)
-    match = cache.check_entry(dataset)
+    key = get_key(path)
+    match = cache.check_entry(key)
     if not match is None:
         match.terminate()
-    return redirect(url_for("do_view", path=path), code=302)
+    qs = request.query_string.decode()
+    return redirect(url_for("do_view", path=path) + (f'?{qs}' if len(qs) > 0 else ''), code=302)
 
 @app.route("/terminate/<path:path>", methods=["GET"])
 def do_terminate(path):
-    dataset = get_dataset(path)
-    match = cache.check_entry(dataset)
+    key = get_key(path)
+    match = cache.check_entry(key)
     if not match is None:
         match.terminate()
     return redirect(url_for("do_GET_status"), code=302)
@@ -232,7 +259,7 @@ def main():
     background_thread.start()
 
     app.launchtime = current_time_stamp()
-    app.run(host="0.0.0.0", port=5005, debug=False)
+    app.run(host="0.0.0.0", port=env.gateway_port, debug=False)
 
 
 if __name__ == "__main__":

cellxgene_gateway/path_util.py

@@ -13,37 +13,83 @@ from flask_api import status
 
 from cellxgene_gateway import env
 from cellxgene_gateway.cellxgene_exception import CellxgeneException
+from cellxgene_gateway.dir_util import make_h5ad
+from cellxgene_gateway.cache_key import CacheKey
 
-
-def get_dataset(path):
+def get_key(path):
     if path == "/" or path == "":
         raise CellxgeneException(
             "No matching dataset found.", status.HTTP_404_NOT_FOUND
         )
 
     trimmed = path[:-1] if path[-1] == "/" else path
-
     try:
-        get_file_path(trimmed)
-        return trimmed
+        # valid paths come in three forms:
+        if trimmed.endswith('.h5ad') and data_file_exists(trimmed):
+            # 1) somedir/dataset.h5ad: a dataset
+            return CacheKey(trimmed, trimmed, None)
+        elif trimmed.endswith('.csv'):
+
+            # 2) somedir/dataset_annotations/saldaal1-T5HMVBNV.csv : an actual annotations file.
+            annotations_dir = os.path.split(trimmed)[0]
+            dataset = make_h5ad(annotations_dir)
+            if data_file_exists(dataset):
+                data_dir_ensure(annotations_dir)
+                return CacheKey(trimmed, dataset, trimmed)
+        elif trimmed.endswith('_annotations') and data_dir_exists(trimmed):
+            # 3) somedir/dataset_annotations: an annotation directory. The corresponding h5ad must exist, but the directory may not.
+            dataset = make_h5ad(trimmed)
+            if data_file_exists(dataset):
+                return CacheKey(trimmed, dataset, '')
     except CellxgeneException:
-        split = os.path.split(trimmed)
-        return get_dataset(split[0])
+        pass
+    split = os.path.split(trimmed)
+    return get_key(split[0])
 
-
-def validate_path(file_path):
+def validate_exists(file_path):
     if not os.path.exists(file_path):
         raise CellxgeneException(
             "File does not exist: " + file_path, status.HTTP_400_BAD_REQUEST
         )
+
+def validate_is_file(file_path):
+    validate_exists(file_path)
     if not os.path.isfile(file_path):
         raise CellxgeneException(
             "Path is not file: " + file_path, status.HTTP_400_BAD_REQUEST
         )
     return
+def validate_is_dir(file_path):
+    validate_exists(file_path)
+    if not os.path.isdir(file_path):
+        raise CellxgeneException(
+            "Path is not dir: " + file_path, status.HTTP_400_BAD_REQUEST
+        )
+    return
 
-
-def get_file_path(dataset):
+def data_file_exists(dataset):
     file_path = os.path.join(env.cellxgene_data, dataset)
-    validate_path(file_path)
+    validate_is_file(file_path)
+    return True
+def data_dir_exists(dataset):
+    file_path = os.path.join(env.cellxgene_data, dataset)
+    validate_is_dir(file_path)
+    return True
+def data_dir_ensure(dataset):
+    file_path = os.path.join(env.cellxgene_data, dataset)
+    if not os.path.exists(file_path):
+        os.makedirs(file_path)
+
+def get_file_path(key):
+    dataset = key.dataset
+    file_path = os.path.join(env.cellxgene_data, dataset)
+    validate_is_file(file_path)
+    return file_path
+
+def get_annotation_file_path(key):
+    if key.annotation_file is None:
+        return None
+    if key.annotation_file == '':
+        return ''
+    file_path = os.path.join(env.cellxgene_data, key.annotation_file)
     return file_path

cellxgene_gateway/process_exception.py

@@ -9,13 +9,13 @@
 
 
 class ProcessException(Exception):
-    def __init__(self, message, stdout, stderr, http_status, dataset):
+    def __init__(self, message, stdout, stderr, http_status, key):
         Exception.__init__(self)
         self.message = message
         self.stdout = stdout
         self.stderr = stderr
         self.http_status = http_status
-        self.dataset = dataset
+        self.key = key
 
     @classmethod
     def from_cache_entry(cls, cache_entry):
@@ -24,5 +24,5 @@ class ProcessException(Exception):
             cache_entry.all_output,
             cache_entry.stderr,
             cache_entry.http_status,
-            cache_entry.dataset,
+            cache_entry.key,
         )

cellxgene_gateway/prune_process_cache.py

@@ -34,7 +34,7 @@ class PruneProcessCache:
         
         for process in processes_to_delete:
             try:
-                logger.info(f"pruning process {process.pid} ({process.dataset})")
+                logger.info(f"pruning process {process.pid} ({process.key.dataset})")
                 self.cache.prune(process)
             except Exception:
                 logger.exception("failed to prune process {process.pid} ({process.dataset})")

cellxgene_gateway/static/js/annotation.js

@@ -0,0 +1,19 @@
+// neandertal javascript
+const new_annotation_callback = (() =>{
+    const suffix = `.csv`;
+    return (e) => {
+        e.preventDefault();
+        const el = $(e.target);
+        const href = el.attr('href');
+        const base = prompt(`Name your annotations collection\nnote: the suffix "${suffix}" will be appended`);
+        if (base !== null && base.length > 0) {
+            if (/^[0-9a-zA-Z_]+$/.test(base)) {
+                window.location = `${href}/${base}${suffix}`;
+            } else {
+                alert("Error: name must match ^[0-9a-zA-Z_]+$\nthat is, only numbers, letters and underscore are allowed")
+            }
+        }
+        return false;
+    }
+})()
+

cellxgene_gateway/subprocess_backend.py

@@ -11,21 +11,32 @@ import logging
 import subprocess
 
 from flask_api import status
-
+from cellxgene_gateway.env import enable_annotations, enable_backed_mode
 from cellxgene_gateway.process_exception import ProcessException
+from cellxgene_gateway.dir_util import make_annotations
+from cellxgene_gateway.path_util import get_file_path, get_annotation_file_path
 
 
 class SubprocessBackend:
     def __init__(self):
         pass
 
-    def create_cmd(self, cellxgene_loc, file_path, port, scripts):
-
+    def create_cmd(self, cellxgene_loc, file_path, port, scripts, annotation_file_path):
+        if enable_annotations and not annotation_file_path is None:
+            if annotation_file_path == "":
+                extra_args = f" --annotations-dir {make_annotations(file_path)}"
+            else:
+                extra_args = f" --annotations-file {annotation_file_path}"
+        else:
+            extra_args = " --disable-annotations"
+        if enable_backed_mode:
+            extra_args += " --backed"
         cmd = (
             f"yes | {cellxgene_loc} launch {file_path}"
             + " --port "
             + str(port)
             + " --host 127.0.0.1"
+            + extra_args
         )
 
         for s in scripts:
@@ -36,7 +47,7 @@ class SubprocessBackend:
     def launch(self, cellxgene_loc, scripts, cache_entry):
 
         cmd = self.create_cmd(
-            cellxgene_loc, cache_entry.file_path, cache_entry.port, scripts
+            cellxgene_loc, get_file_path(cache_entry.key), cache_entry.port, scripts, get_annotation_file_path(cache_entry.key)
         )
         logging.getLogger("cellxgene_gateway").info(f"launching {cmd}")
         process = subprocess.Popen(

cellxgene_gateway/templates/cache_status.html

@@ -29,6 +29,7 @@
           <tr>
                <th>PID</th>
                <th>dataset</th>
+               <th>annotation_file</th>
                <th>port</th>
                <th>launchtime</th>
                <th>last access</th>
@@ -42,7 +43,8 @@
 	{% for entry in entry_list %}
           <tr>
                <td>{{ entry.pid }}</td>
-               <td><a href="{{ url_for('do_view', path=entry.dataset) }}">{{ entry.dataset }}</a></td>
+               <td><a href="{{ url_for('do_view', path=entry.key.pathpart) }}">{{ entry.key.dataset }}</a></td>
+               <td>{{ entry.key.annotation_file }}</td>
                <td>{{ entry.port }}</td>
                <td class="timestamp">{{ entry.launchtime }}</td>
                <td class="timestamp">{{ entry.timestamp }}</td>
@@ -51,7 +53,7 @@
                <td>{{ entry.http_status }}</td>
                <td>
                  {% if entry.status == 'loaded' %}
-                   <a href="{{ url_for('do_terminate', path=entry.dataset) }}"> terminate </a>
+                   <a href="{{ url_for('do_terminate', path=entry.key.pathpart) }}"> terminate </a>
                  {% endif %}
                </td>
           </tr>

cellxgene_gateway/templates/filecrawl.html

@@ -16,19 +16,36 @@
      <link rel="icon" type="image/png" href="{{ url_for('static', filename='nibr.ico') }}">
 	{% for script in extra_scripts %}
 	<script src="{{ script }}"></script>
-	{% endfor %}
+     {% endfor %}
+     <script src="{{ url_for('static', filename='js/annotation.js') }}"></script>
      <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css" integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO" crossorigin="anonymous">
 </head>
 <body>
      <header class="navbar navbar-expand navbar-dark flex-column flex-md-row bd-navbar">
-       <h3>Cellxgene Gateway - FILE CRAWLER</h3>
+          {% if path %}
+          <h3>Cellxgene Gateway - {{ path }}</h3>
+          {% else %}
+          <h3>Cellxgene Gateway - FILE CRAWLER</h3>
+          {% endif %}
      </header>
-     <br>
 
      <h4>Please click on a dataset to view it in Cellxgene Server.</h4>
 
-     <br>
      {{ rendered_html|safe }}
-
+     <p>
+          Navigation:
+     <ul>
+     {% if path %}
+     <li><a href="/filecrawl.html">top level</a></li>
+     {% else %}
+     {% endif %}
+     <li><a href="/">homepage</a></li>
+     </ul>
+     </p>
+     <script>
+          $(() => {
+               $("a.new").click(new_annotation_callback);
+          })
+     </script>
 </body>
 </html>

cellxgene_gateway/templates/loading.html

@@ -44,9 +44,13 @@
      </a>
      </div>
      <script>
+          var count = 0;
           window.setInterval(function(){
                var dots = document.getElementById('dots');
                dots.textContent = dots.textContent + '.';
+               if (count++ > 5) {
+                    window.location.reload();
+               }
           }, 1000);
      </script>
 </body>

environment.yml

@@ -8,4 +8,4 @@ dependencies:
 - psutil
 - pip:
   - flask-api
-  - cellxgene
+  - cellxgene>=0.15

requirements.txt

@@ -1,4 +1,4 @@
-cellxgene
+cellxgene>=0.15
 flask
 flask_api
 psutil

run.sh.example

@@ -1,8 +1,5 @@
 export CELLXGENE_LOCATION=$(pwd)/.cellxgene-gateway/bin/cellxgene
 export CELLXGENE_DATA=../cellxgene_data
-export DEPLOYMENT_ENV=dev
-export GATEWAY_HOST=localhost:5005
-export GATEWAY_PROTOCOL=http
 export GATEWAY_IP=127.0.0.1
 
 #Once these are set, you run like a normal Flask app

setup.cfg

@@ -0,0 +1,2 @@
+[metadata]
+description-file = README.md

setup.py

@@ -1,6 +1,23 @@
 import os
-from setuptools import setup
+import codecs
+from setuptools import find_packages, setup
+import sys
 
+if sys.version_info < (3,6):
+    sys.exit('Sorry, Python < 3.6 is not supported')
+
+def read(rel_path):
+    here = os.path.abspath(os.path.dirname(__file__))
+    with codecs.open(os.path.join(here, rel_path), 'r') as fp:
+        return fp.read()
+
+def get_version(rel_path):
+    for line in read(rel_path).splitlines():
+        if line.startswith('__version__'):
+            delim = '"' if '"' in line else "'"
+            return line.split(delim)[1]
+    else:
+        raise RuntimeError("Unable to find version string.")
 
 def parse_requirements():
     reqs = []
@@ -9,6 +26,8 @@ def parse_requirements():
             reqs.append(l.strip("\n"))
     return reqs
 
+with open("README.md", "r") as fh:
+    long_description = fh.read()
 
 install_reqs = parse_requirements()
 
@@ -16,11 +35,13 @@ setup(
     # mandatory
     name="cellxgene-gateway",
     # mandatory
-    version="0.1",
+    version=get_version("cellxgene_gateway/__init__.py"),
     # mandatory
     author="Niket Patel, Yohann Potier, Alok Saldanha",
     author_email="alok.saldanha@novartis.com",
     description=("Cellxgene Gateway"),
+    long_description=long_description,
+    long_description_content_type="text/markdown",
     license="MIT",
     keywords="visualization, genomics",
     url="http://github.com/Novartis/cellxgene-gateway",
@@ -31,10 +52,11 @@ setup(
             "static/nibr.ico",
             "templates/*.html"
     ]},
-    data_files=[('', ['Readme.md', 'LICENSE.txt'])],
+    data_files=[('', ['README.md', 'LICENSE'])],
     install_requires=install_reqs,
     entry_points={
         "console_scripts": ["cellxgene-gateway=cellxgene_gateway.gateway:main"]
     },
     classifiers=["Topic :: Scientific/Engineering :: Visualization"],
+    python_requires='>=3.6',
 )