#81 added unit test for gene sets

This adds the flag `GATEWAY_ENABLE_GENE_SETS` to enable support for gene
sets.  To simplify implementation, activating this flag also activates
`GATEWAY_ENABLE_ANNOTATIONS`.  The gene sets are saved in a file that
has the same name as the annotations `csv` but with `_gene_sets`
appended to the file name (before the extension).  This file is hidden
in filecrawler, and the gene sets are loaded when the associated
annotations file is loaded.

If the annotations file is missing, then an Exception is raised.

I have updated one unit test to make it expect
`--disable-gene-sets-save` in the default case (i.e. if
`GATEWAY_ENABLE_ANNOTATIONS = 0`).  All units tests pass.

I have updated the README to document `GATEWAY_ENABLE_GENE_SETS`.

.github/workflows/pr-checks.yaml

@@ -6,7 +6,7 @@ on: [push, pull_request]
 
 jobs:
   black:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
         name: Checkout repository
@@ -25,7 +25,7 @@ jobs:
           black . --check
   # This job is copied over from `deploy.yaml`
   run-tests:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
 

Changelog.md

@@ -1,3 +1,7 @@
+# 0.3.11
+
+* #81 added support for gene sets
+
 # 0.3.10
 
 * #65 Added GATEWAY_EXPIRE_SECONDS to set how long cellxgene servers can remain idle before being terminated.

README.md

@@ -75,7 +75,7 @@ Optional environment variables:
 * `GATEWAY_PORT` - local port that the gateway should bind to, defaults to 5005
 * `GATEWAY_EXPIRE_SECONDS` - time in seconds that a cellxgene process will remain idle before being terminated. Defaults to 3600 (one hour)
 * `GATEWAY_EXTRA_SCRIPTS` - JSON array of script paths, will be embedded into each page and forwarded with `--scripts` to cellxgene server
-* `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations. 
+* `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations and gene sets.
 * `GATEWAY_ENABLE_BACKED_MODE` - Set to `true` or to `1` to load AnnData in file-backed mode. This saves memory and speeds up launch time but may reduce overall performance.
 * `GATEWAY_LOG_LEVEL` - default is `INFO`. set to `DEBUG` to increase logging and to `WARNING` to decrease logging.
 * `S3_ENABLE_LISTINGS_CACHE` - Set to `true` or to `1` to cache listings of S3 folders for performance. If the cache becomes stale, set `filecrawl.html?refresh=true` query parameter to refresh the cache.
@@ -186,21 +186,6 @@ pip install isort flake8 black
 isort -rc . # rc means recursive, and was deprecated in dev version of isort
 black .
 ```
-## Dependency management
-
-We use the following files for dependency management:
-
-* environment.yml - specifies a conda environment sufficient to run the packages
-* setup.cfg - lists static dependency information, both minimal and extra dependencies
-* setup.py - dynamically generated package information
-* requirements.txt - simple wrapper invoking setup.py
-
-For more details, see the folloiwng links:
-
-* https://towardsdatascience.com/setuptools-python-571e7d5500f2
-* https://towardsdatascience.com/requirements-vs-setuptools-python-ae3ee66e28af
-
-
 
 # Getting Help
 

cellxgene_gateway/cache_entry.py

@@ -58,7 +58,6 @@ class CacheEntry:
 
     @classmethod
     def for_key(cls, key, port):
-
         return cls(
             None,
             key,

cellxgene_gateway/flask_util.py

@@ -26,8 +26,8 @@ def url(endpoint, descriptor, source_name):
 
 
 def view_url(descriptor, source_name):
-    return url("gateway_blueprint.do_view", descriptor, source_name)
+    return url("do_view", descriptor, source_name)
 
 
 def relaunch_url(descriptor, source_name):
-    return url("gateway_blueprint.do_relaunch", descriptor, source_name)
+    return url("do_relaunch", descriptor, source_name)

cellxgene_gateway/gateway.py

@@ -1,15 +1,45 @@
+# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
+# under the Apache License, Version 2.0 (the "License"); you may not use
+# this file except in compliance with the License. You may obtain a copy
+# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
+# required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+# the specific language governing permissions and limitations under the License.
+# import BaseHTTPServer
+import json
 import logging
 import os
+import urllib.parse
+from threading import Lock, Thread
 
-import typer
-from flask import Flask
+from flask import (
+    Flask,
+    make_response,
+    redirect,
+    render_template,
+    request,
+    send_from_directory,
+    url_for,
+)
 from werkzeug.middleware.proxy_fix import ProxyFix
 
-from cellxgene_gateway import env, flask_util, gateway_blueprint
+from cellxgene_gateway import env, flask_util
+from cellxgene_gateway.backend_cache import BackendCache
+from cellxgene_gateway.cache_entry import CacheEntryStatus
+from cellxgene_gateway.cache_key import CacheKey
+from cellxgene_gateway.cellxgene_exception import CellxgeneException
+from cellxgene_gateway.extra_scripts import get_extra_scripts
+from cellxgene_gateway.filecrawl import render_item_source
+from cellxgene_gateway.process_exception import ProcessException
+from cellxgene_gateway.prune_process_cache import PruneProcessCache
 from cellxgene_gateway.util import current_time_stamp
 
 app = Flask(__name__)
 
+item_sources = []
+default_item_source = None
+
 
 def _force_https(app):
     def wrapper(environ, start_response):
@@ -20,6 +50,14 @@ def _force_https(app):
     return wrapper
 
 
+def set_no_cache(resp):
+    resp.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
+    resp.headers["Pragma"] = "no-cache"
+    resp.headers["Expires"] = "0"
+    resp.headers["Cache-Control"] = "public, max-age=0"
+    return resp
+
+
 app.wsgi_app = _force_https(app.wsgi_app)
 if (
     env.proxy_fix_for > 0
@@ -37,8 +75,227 @@ if (
         x_prefix=env.proxy_fix_prefix,
     )
 
+cache = BackendCache()
 
-def main(prometheus: bool = False):
+
+@app.errorhandler(CellxgeneException)
+def handle_invalid_usage(error):
+    message = f"{error.http_status} Error : {error.message}"
+
+    return (
+        render_template(
+            "cellxgene_error.html",
+            extra_scripts=get_extra_scripts(),
+            message=message,
+        ),
+        error.http_status,
+    )
+
+
+@app.errorhandler(ProcessException)
+def handle_invalid_process(error):
+    message = []
+
+    message.append(error.message)
+    message.append(f"{error.http_status} Error.")
+    message.append(f"Stdout: {error.stdout}")
+    message.append(f"Stderr: {error.stderr}")
+
+    return (
+        render_template(
+            "process_error.html",
+            extra_scripts=get_extra_scripts(),
+            message=error.message,
+            http_status=error.http_status,
+            stdout=error.stdout,
+            stderr=error.stderr,
+            relaunch_url=error.key.relaunch_url(),
+            annotation_file=error.key.annotation_descriptor,
+        ),
+        error.http_status,
+    )
+
+
+@app.route("/favicon.ico")
+def favicon():
+    return send_from_directory(
+        os.path.join(app.root_path, "static"),
+        "nibr.ico",
+        mimetype="image/vnd.microsof.icon",
+    )
+
+
+@app.route("/")
+def index():
+    return render_template(
+        "index.html",
+        ip=env.ip,
+        cellxgene_data=env.cellxgene_data,
+        extra_scripts=get_extra_scripts(),
+    )
+
+
+@app.route("/filecrawl.html")
+@app.route("/filecrawl/<path:path>")
+def filecrawl(path=None):
+    source_name = request.args.get("source")
+    sources = (
+        filter(
+            lambda x: x.name == urllib.parse.unquote_plus(source_name),
+            item_sources,
+        )
+        if source_name
+        else item_sources
+    )
+    # loop all data sources --
+    rendered_sources = [
+        render_item_source(item_source, path) for item_source in sources
+    ]  # will we need to make this async in the page???
+    rendered_html = "\n".join(rendered_sources)
+
+    resp = make_response(
+        render_template(
+            "filecrawl.html",
+            extra_scripts=get_extra_scripts(),
+            rendered_html=rendered_html,
+            path=path,
+        )
+    )
+    set_no_cache(resp)
+    return resp
+
+
+entry_lock = Lock()
+
+
+def matching_source(source_name):
+    if source_name is None:
+        source_name = default_item_source.name
+    matching = [i for i in item_sources if i.name == source_name]
+    if len(matching) != 1:
+        raise Exception(f"Could not find matching item source {source_name}")
+    source = matching[0]
+    return source
+
+
+@app.route(
+    "/source/<path:source_name>/view/<path:path>",
+    methods=["GET", "PUT", "POST"],
+)
+@app.route("/view/<path:path>", methods=["GET", "PUT", "POST"])
+def do_view(path, source_name=None):
+    source = matching_source(source_name)
+    match = cache.check_path(source, path)
+
+    if match is None:
+        lookup = source.lookup(path)
+        if lookup is None:
+            raise CellxgeneException(
+                f"Could not find item for path {path} in source {source.name}",
+                404,
+            )
+        key = CacheKey.for_lookup(source, lookup)
+        print(
+            f"view path={path}, source_name={source_name}, dataset={key.file_path}, annotation_file= {key.annotation_file_path}, key={key.descriptor}, source={key.source_name}"
+        )
+        with entry_lock:
+            match = cache.check_entry(key)
+            if match is None:
+                uascripts = get_extra_scripts()
+                match = cache.create_entry(key, uascripts)
+
+    match.timestamp = current_time_stamp()
+
+    if (
+        match.status == CacheEntryStatus.loaded
+        or match.status == CacheEntryStatus.loading
+    ):
+        if source.is_authorized(match.key.descriptor):
+            return match.serve_content(path)
+        else:
+            raise CellxgeneException("User not authorized to access this data", 403)
+    elif match.status == CacheEntryStatus.error:
+        raise ProcessException.from_cache_entry(match)
+
+
+@app.route("/cache_status", methods=["GET"])
+def do_GET_status():
+    return render_template(
+        "cache_status.html",
+        entry_list=cache.entry_list,
+        extra_scripts=get_extra_scripts(),
+    )
+
+
+@app.route("/cache_status.json", methods=["GET"])
+def do_GET_status_json():
+    return json.dumps(
+        {
+            "launchtime": app.launchtime,
+            "entry_list": [
+                {
+                    "dataset": entry.key.dataset,
+                    "annotation_file": entry.key.annotation_file,
+                    "launchtime": entry.launchtime,
+                    "last_access": entry.timestamp,
+                    "status": entry.status,
+                }
+                for entry in cache.entry_list
+            ],
+        }
+    )
+
+
+@app.route("/relaunch/<path:path>", methods=["GET"])
+def do_relaunch(path):
+    source_name = request.args.get("source_name") or default_item_source.name
+    source = matching_source(source_name)
+    key = CacheKey.for_lookup(source, source.lookup(path))
+    match = cache.check_entry(key)
+    if not match is None:
+        match.terminate()
+    return redirect(
+        key.view_url,
+        code=302,
+    )
+
+
+@app.route("/terminate/<path:path>", methods=["GET"])
+def do_terminate(path):
+    source_name = request.args.get("source_name") or default_item_source.name
+    source = matching_source(source_name)
+    key = CacheKey.for_lookup(source, source.lookup(path))
+    match = cache.check_entry(key)
+    if not match is None:
+        match.terminate()
+    return redirect(url_for("do_GET_status"), code=302)
+
+
+@app.route("/metadata/ip_address", methods=["GET"])
+def ip_address():
+    resp = make_response(env.ip)
+    return set_no_cache(resp)
+
+
+def launch():
+    env.validate()
+    if not item_sources or not len(item_sources):
+        raise Exception("No data sources specified for Cellxgene Gateway")
+
+    global default_item_source
+    if default_item_source is None:
+        default_item_source = item_sources[0]
+
+    pruner = PruneProcessCache(cache)
+
+    background_thread = Thread(target=pruner)
+    background_thread.start()
+
+    app.launchtime = current_time_stamp()
+    app.run(host="0.0.0.0", port=env.gateway_port, debug=False)
+
+
+def main():
     logging.basicConfig(
         level=env.log_level,
         format="%(asctime)s:%(name)s:%(levelname)s:%(message)s",
@@ -49,33 +306,19 @@ def main(prometheus: bool = False):
     if cellxgene_bucket is not None:
         from cellxgene_gateway.items.s3.s3item_source import S3ItemSource
 
-        gateway_blueprint.item_sources.append(S3ItemSource(cellxgene_bucket, name="s3"))
+        item_sources.append(S3ItemSource(cellxgene_bucket, name="s3"))
         default_item_source = "s3"
     if cellxgene_data is not None:
         from cellxgene_gateway.items.file.fileitem_source import FileItemSource
 
-        gateway_blueprint.item_sources.append(
-            FileItemSource(cellxgene_data, name="local")
-        )
+        item_sources.append(FileItemSource(cellxgene_data, name="local"))
         default_item_source = "local"
-    if len(gateway_blueprint.item_sources) == 0:
+    if len(item_sources) == 0:
         raise Exception("Please specify CELLXGENE_DATA or CELLXGENE_BUCKET")
-    flask_util.include_source_in_url = len(gateway_blueprint.item_sources) > 1
+    flask_util.include_source_in_url = len(item_sources) > 1
 
-    if prometheus:
-        from cellxgene_gateway.prometheus import add_metrics
-
-        add_metrics(app)
-    app.register_blueprint(gateway_blueprint.gateway_blueprint)
-    gateway_blueprint.launch()
-
-    app.launchtime = current_time_stamp()
-    app.run(host="0.0.0.0", port=env.gateway_port, debug=False)
-
-
-def run():
-    typer.run(main)
+    launch()
 
 
 if __name__ == "__main__":
-    run()
+    main()

cellxgene_gateway/gateway_blueprint.py

@@ -1,266 +0,0 @@
-# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
-# under the Apache License, Version 2.0 (the "License"); you may not use
-# this file except in compliance with the License. You may obtain a copy
-# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
-# required by applicable law or agreed to in writing, software distributed
-# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
-# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
-# the specific language governing permissions and limitations under the License.
-# import BaseHTTPServer
-import json
-import os
-import urllib.parse
-from threading import Lock, Thread
-
-from flask import (
-    Blueprint,
-    current_app,
-    make_response,
-    redirect,
-    render_template,
-    request,
-    send_from_directory,
-    url_for,
-)
-
-from cellxgene_gateway import env
-from cellxgene_gateway.backend_cache import BackendCache
-from cellxgene_gateway.cache_entry import CacheEntryStatus
-from cellxgene_gateway.cache_key import CacheKey
-from cellxgene_gateway.cellxgene_exception import CellxgeneException
-from cellxgene_gateway.extra_scripts import get_extra_scripts
-from cellxgene_gateway.filecrawl import render_item_source
-from cellxgene_gateway.process_exception import ProcessException
-from cellxgene_gateway.prune_process_cache import PruneProcessCache
-from cellxgene_gateway.util import current_time_stamp
-
-gateway_blueprint = Blueprint("gateway_blueprint", __name__)
-item_sources = []
-default_item_source = None
-
-
-def set_no_cache(resp):
-    resp.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
-    resp.headers["Pragma"] = "no-cache"
-    resp.headers["Expires"] = "0"
-    resp.headers["Cache-Control"] = "public, max-age=0"
-    return resp
-
-
-cache = BackendCache()
-
-
-@gateway_blueprint.errorhandler(CellxgeneException)
-def handle_invalid_usage(error):
-
-    message = f"{error.http_status} Error : {error.message}"
-
-    return (
-        render_template(
-            "cellxgene_error.html",
-            extra_scripts=get_extra_scripts(),
-            message=message,
-        ),
-        error.http_status,
-    )
-
-
-@gateway_blueprint.errorhandler(ProcessException)
-def handle_invalid_process(error):
-
-    message = []
-
-    message.append(error.message)
-    message.append(f"{error.http_status} Error.")
-    message.append(f"Stdout: {error.stdout}")
-    message.append(f"Stderr: {error.stderr}")
-
-    return (
-        render_template(
-            "process_error.html",
-            extra_scripts=get_extra_scripts(),
-            message=error.message,
-            http_status=error.http_status,
-            stdout=error.stdout,
-            stderr=error.stderr,
-            relaunch_url=error.key.relaunch_url(),
-            annotation_file=error.key.annotation_descriptor,
-        ),
-        error.http_status,
-    )
-
-
-@gateway_blueprint.route("/favicon.ico")
-def favicon():
-    return send_from_directory(
-        os.path.join(current_app.root_path, "static"),
-        "nibr.ico",
-        mimetype="image/vnd.microsof.icon",
-    )
-
-
-@gateway_blueprint.route("/")
-def index():
-    return render_template(
-        "index.html",
-        ip=env.ip,
-        cellxgene_data=env.cellxgene_data,
-        extra_scripts=get_extra_scripts(),
-    )
-
-
-@gateway_blueprint.route("/filecrawl.html")
-@gateway_blueprint.route("/filecrawl/<path:path>")
-def filecrawl(path=None):
-    source_name = request.args.get("source")
-    sources = (
-        filter(
-            lambda x: x.name == urllib.parse.unquote_plus(source_name),
-            item_sources,
-        )
-        if source_name
-        else item_sources
-    )
-    # loop all data sources --
-    rendered_sources = [
-        render_item_source(item_source, path) for item_source in sources
-    ]  # will we need to make this async in the page???
-    rendered_html = "\n".join(rendered_sources)
-
-    resp = make_response(
-        render_template(
-            "filecrawl.html",
-            extra_scripts=get_extra_scripts(),
-            rendered_html=rendered_html,
-            path=path,
-        )
-    )
-    set_no_cache(resp)
-    return resp
-
-
-entry_lock = Lock()
-
-
-def matching_source(source_name):
-    if source_name is None:
-        source_name = default_item_source.name
-    matching = [i for i in item_sources if i.name == source_name]
-    if len(matching) != 1:
-        raise Exception(f"Could not find matching item source {source_name}")
-    source = matching[0]
-    return source
-
-
-@gateway_blueprint.route(
-    "/source/<path:source_name>/view/<path:path>",
-    methods=["GET", "PUT", "POST"],
-)
-@gateway_blueprint.route("/view/<path:path>", methods=["GET", "PUT", "POST"])
-def do_view(path, source_name=None):
-    source = matching_source(source_name)
-    match = cache.check_path(source, path)
-
-    if match is None:
-        lookup = source.lookup(path)
-        if lookup is None:
-            raise CellxgeneException(
-                f"Could not find item for path {path} in source {source.name}",
-                404,
-            )
-        key = CacheKey.for_lookup(source, lookup)
-        print(
-            f"view path={path}, source_name={source_name}, dataset={key.file_path}, annotation_file= {key.annotation_file_path}, key={key.descriptor}, source={key.source_name}"
-        )
-        with entry_lock:
-            match = cache.check_entry(key)
-            if match is None:
-                uascripts = get_extra_scripts()
-                match = cache.create_entry(key, uascripts)
-
-    match.timestamp = current_time_stamp()
-
-    if (
-        match.status == CacheEntryStatus.loaded
-        or match.status == CacheEntryStatus.loading
-    ):
-        if source.is_authorized(match.key.descriptor):
-            return match.serve_content(path)
-        else:
-            raise CellxgeneException("User not authorized to access this data", 403)
-    elif match.status == CacheEntryStatus.error:
-        raise ProcessException.from_cache_entry(match)
-
-
-@gateway_blueprint.route("/cache_status", methods=["GET"])
-def do_GET_status():
-    return render_template(
-        "cache_status.html",
-        entry_list=cache.entry_list,
-        extra_scripts=get_extra_scripts(),
-    )
-
-
-@gateway_blueprint.route("/cache_status.json", methods=["GET"])
-def do_GET_status_json():
-    return json.dumps(
-        {
-            "launchtime": current_app.launchtime,
-            "entry_list": [
-                {
-                    "dataset": entry.key.dataset,
-                    "annotation_file": entry.key.annotation_file,
-                    "launchtime": entry.launchtime,
-                    "last_access": entry.timestamp,
-                    "status": entry.status,
-                }
-                for entry in cache.entry_list
-            ],
-        }
-    )
-
-
-@gateway_blueprint.route("/relaunch/<path:path>", methods=["GET"])
-def do_relaunch(path):
-    source_name = request.args.get("source_name") or default_item_source.name
-    source = matching_source(source_name)
-    key = CacheKey.for_lookup(source, source.lookup(path))
-    match = cache.check_entry(key)
-    if not match is None:
-        match.terminate()
-    return redirect(
-        key.view_url,
-        code=302,
-    )
-
-
-@gateway_blueprint.route("/terminate/<path:path>", methods=["GET"])
-def do_terminate(path):
-    source_name = request.args.get("source_name") or default_item_source.name
-    source = matching_source(source_name)
-    key = CacheKey.for_lookup(source, source.lookup(path))
-    match = cache.check_entry(key)
-    if not match is None:
-        match.terminate()
-    return redirect(url_for("gateway_blueprint.do_GET_status"), code=302)
-
-
-@gateway_blueprint.route("/metadata/ip_address", methods=["GET"])
-def ip_address():
-    resp = make_response(env.ip)
-    return set_no_cache(resp)
-
-
-def launch():
-    env.validate()
-    if not item_sources or not len(item_sources):
-        raise Exception("No data sources specified for Cellxgene Gateway")
-
-    global default_item_source
-    if default_item_source is None:
-        default_item_source = item_sources[0]
-
-    pruner = PruneProcessCache(cache)
-
-    background_thread = Thread(target=pruner)
-    background_thread.start()

cellxgene_gateway/items/file/fileitem_source.py

@@ -35,6 +35,11 @@ class FileItemSource(ItemSource):
     def name(self):
         return self._name or f"Files:{self.base_path}"
 
+    def is_gene_set(self, path: str) -> bool:
+        return ("_gene_sets" in path or "-gene-sets" in path) and path.endswith(
+            self.annotation_file_suffix
+        )
+
     def is_h5ad_file(self, path: str) -> bool:
         return path.endswith(self.h5ad_suffix) and os.path.isfile(path)
 
@@ -180,6 +185,7 @@ class FileItemSource(ItemSource):
                 self.make_fileitem_from_path(annotation, annotations_subpath, True)
                 for annotation in sorted(os.listdir(annotations_fullpath))
                 if annotation.endswith(self.annotation_file_suffix)
+                and not self.is_gene_set(annotation)
                 and os.path.isfile(os.path.join(annotations_fullpath, annotation))
             ]
         else:

cellxgene_gateway/prometheus.py

@@ -1,9 +0,0 @@
-from prometheus_flask_exporter import PrometheusMetrics
-
-from cellxgene_gateway import __version__
-
-
-def add_metrics(app):
-    metrics = PrometheusMetrics(app)
-    metrics.info("app_info", "Application info", version=__version__)
-    return metrics

cellxgene_gateway/subprocess_backend.py

@@ -14,7 +14,11 @@ from flask_api import status
 
 from cellxgene_gateway.cache_entry import CacheEntryStatus
 from cellxgene_gateway.dir_util import make_annotations
-from cellxgene_gateway.env import cellxgene_args, enable_annotations, enable_backed_mode
+from cellxgene_gateway.env import (
+    cellxgene_args,
+    enable_annotations,
+    enable_backed_mode,
+)
 from cellxgene_gateway.process_exception import ProcessException
 
 logger = logging.getLogger(__name__)
@@ -30,8 +34,11 @@ class SubprocessBackend:
                 extra_args = f" --annotations-dir {make_annotations(file_path)}"
             else:
                 extra_args = f" --annotations-file {annotation_file_path}"
+                gene_sets_file_path = annotation_file_path[:-4] + "_gene_sets.csv"
+                extra_args += f" --gene-sets-file {gene_sets_file_path}"
         else:
             extra_args = " --disable-annotations"
+            extra_args += " --disable-gene-sets-save"
         if enable_backed_mode:
             extra_args += " --backed"
         if not cellxgene_args is None:

cellxgene_gateway/templates/cache_status.html

@@ -61,7 +61,7 @@
                     <td>
                          {% if entry.status.name == 'loaded' %}
                          <a
-                              href="{{ url_for('gateway_blueprint.do_terminate', path=entry.key.descriptor, source_name=entry.key.source_name) }}">
+                              href="{{ url_for('do_terminate', path=entry.key.descriptor, source_name=entry.key.source_name) }}">
                               terminate </a>
                          {% endif %}
                     </td>

cellxgene_gateway/templates/cellxgene_error.html

@@ -28,11 +28,11 @@
           
      <h4>{{ message }}</h4>
 
-     <a href="{{ url_for('gateway_blueprint.filecrawl') }}">
+     <a href="{{ url_for('filecrawl') }}">
           Please click here to be redirected to the file directory.
      </a>
      <br>
-     <a href="{{ url_for('gateway_blueprint.index') }}">
+     <a href="{{ url_for('index') }}">
           Please click here to return to the homepage.
      </a>
      </div>

cellxgene_gateway/templates/filecrawl.html

@@ -36,10 +36,10 @@
           Navigation:
      <ul>
      {% if path %}
-     <li><a href="{{ url_for('gateway_blueprint.filecrawl') }}">top level</a></li>
+     <li><a href="{{ url_for('filecrawl') }}">top level</a></li>
      {% else %}
      {% endif %}
-     <li><a href="{{ url_for('gateway_blueprint.index') }}">homepage</a></li>
+     <li><a href="{{ url_for('index') }}">homepage</a></li>
      </ul>
      </p>
      <script>

cellxgene_gateway/templates/index.html

@@ -35,12 +35,12 @@
 		Links:	
 	</h1>
 	<div class="list-group" style="width:50%;padding-left:65px">
-	  <a href="{{ url_for('gateway_blueprint.filecrawl') }}" class="list-group-item list-group-item-action">
+	  <a href="{{ url_for('filecrawl') }}" class="list-group-item list-group-item-action">
 		  <u>File Crawler: Allows you to view all uploaded data.</u></a>
 		  
 	</div>
 	<div class="list-group" style="width:50%;padding-left:65px">
-		<a href="{{ url_for('gateway_blueprint.do_GET_status') }}" class="list-group-item list-group-item-action">
+		<a href="{{ url_for('do_GET_status') }}" class="list-group-item list-group-item-action">
 			<u>Cache Status: view status of launched cellxgene servers.</u></a>
 	</div>
 	

cellxgene_gateway/templates/loading.html

@@ -35,11 +35,11 @@
           The page will refresh shortly.
      </p>
 
-     <a href="{{ url_for('gateway_blueprint.filecrawl') }}">
+     <a href="{{ url_for('filecrawl') }}">
           Please click here to be redirected to the file directory.
      </a>
      <br>
-     <a href="{{ url_for('gateway_blueprint.index') }}">
+     <a href="{{ url_for('index') }}">
           Please click here to return to the homepage.
      </a>
      </div>

cellxgene_gateway/templates/process_error.html

@@ -39,10 +39,10 @@
           <li><a href="{{ relaunch_url }}">
                     Attempt to relaunch the cellxgene server.
                </a></li>
-          <li><a href="{{ url_for('gateway_blueprint.filecrawl') }}">
+          <li><a href="{{ url_for('filecrawl') }}">
                Return to the file directory.
           </a></li>
-          <li><a href="{{ url_for('gateway_blueprint.index') }}">
+          <li><a href="{{ url_for('index') }}">
                Return to the homepage.
           </a></li>
      </ul>

environment.yml

@@ -7,7 +7,6 @@ dependencies:
 - flask
 - psutil
 - black
-- typer
 - twine
 - isort
 - coverage
@@ -17,4 +16,3 @@ dependencies:
   - flask-api
   - werkzeug
   - cellxgene
-  - prometheus-flask-exporter

requirements.txt

@@ -1,9 +1,6 @@
- # requirements.txt
- #
- # installs dependencies from ./setup.py, and the package itself,
- # in editable mode
- -e .[prometheus]
-
- # (the -e above is optional). you could also just install the package
- # normally with just the line below (after uncommenting)
- # .
+cellxgene
+flask
+flask-api
+werkzeug
+psutil
+requests

setup.cfg

@@ -1,30 +1,2 @@
 [metadata]
-description_file = README.md
-description = "Cellxgene Gateway"
-author = "Niket Patel, Yohann Potier, Alok Saldanha"
-author_email = "alok.saldanha@novartis.com"
-long_description_content_type="text/markdown"
-license = "MIT"
-keywords ="visualization, genomics"
-url = "http://github.com/Novartis/cellxgene-gateway"
-python_requires = ">=3.6"
-classifier =
-    "Topic :: Scientific/Engineering :: Visualization"
-
-[options]
-install_requires =
-    cellxgene
-    flask
-    flask-api
-    werkzeug
-    psutil
-    requests
-    typer
-
-[options.extras_require]
-prometheus =
-    prometheus-flask-exporter
-
-[entry_points]
-console_scripts =
-    cellxgene-gateway = cellxgene_gateway.cli:run
+description-file = README.md

setup.py

@@ -4,6 +4,9 @@ import sys
 
 from setuptools import find_packages, setup
 
+if sys.version_info < (3, 6):
+    sys.exit("Sorry, Python < 3.6 is not supported")
+
 
 def read(rel_path):
     here = os.path.abspath(os.path.dirname(__file__))
@@ -20,15 +23,33 @@ def get_version(rel_path):
         raise RuntimeError("Unable to find version string.")
 
 
+def parse_requirements():
+    reqs = []
+    with open("requirements.txt", "r") as f:
+        for line in f.readlines():
+            reqs.append(line.strip("\n"))
+    return reqs
+
+
 with open("README.md", "r") as fh:
     long_description = fh.read()
 
+install_reqs = parse_requirements()
+
 setup(
+    # mandatory
+    name="cellxgene-gateway",
     # mandatory
     version=get_version("cellxgene_gateway/__init__.py"),
     # mandatory
+    author="Niket Patel, Yohann Potier, Alok Saldanha",
+    author_email="alok.saldanha@novartis.com",
+    description=("Cellxgene Gateway"),
     long_description=long_description,
-    name="cellxgene-gateway",
+    long_description_content_type="text/markdown",
+    license="MIT",
+    keywords="visualization, genomics",
+    url="http://github.com/Novartis/cellxgene-gateway",
     packages=find_packages(),
     package_data={
         "cellxgene_gateway": [
@@ -39,4 +60,10 @@ setup(
         ]
     },
     data_files=[("", ["README.md", "LICENSE"])],
+    install_requires=install_reqs,
+    entry_points={
+        "console_scripts": ["cellxgene-gateway=cellxgene_gateway.gateway:main"]
+    },
+    classifiers=["Topic :: Scientific/Engineering :: Visualization"],
+    python_requires=">=3.6",
 )

tests/items/s3/test_s3item_source.py

@@ -1,9 +1,7 @@
 import unittest
 from unittest.mock import MagicMock, Mock, patch
 
-from flask import Flask
-
-from cellxgene_gateway.gateway_blueprint import gateway_blueprint
+from cellxgene_gateway.gateway import app
 from cellxgene_gateway.items.item import ItemType
 from cellxgene_gateway.items.s3.s3item import S3Item
 from cellxgene_gateway.items.s3.s3item_source import S3ItemSource
@@ -84,8 +82,6 @@ class TestScanDirectory(unittest.TestCase):
 
         s3func.return_value = S3Mock
         source = S3ItemSource("my-bucket")
-        app = Flask(__name__)
-        app.register_blueprint(gateway_blueprint)
         with app.test_request_context(query_string="refresh=true") as test_context:
             tree = source.scan_directory()
 

tests/test_cache_entry.py

@@ -5,7 +5,7 @@ from flask import Flask
 from cellxgene_gateway import flask_util
 from cellxgene_gateway.cache_entry import CacheEntry, CacheEntryStatus
 from cellxgene_gateway.cache_key import CacheKey
-from cellxgene_gateway.gateway_blueprint import gateway_blueprint
+from cellxgene_gateway.gateway import app
 from cellxgene_gateway.items.file.fileitem import FileItem
 from cellxgene_gateway.items.file.fileitem_source import FileItemSource
 from cellxgene_gateway.items.item import ItemType
@@ -18,8 +18,7 @@ key = CacheKey(
 
 class TestRenderEntry(unittest.TestCase):
     def setUp(self):
-        self.app = Flask(__name__)
-        self.app.register_blueprint(gateway_blueprint)
+        self.app = app
         self.app_context = self.app.test_request_context()
         self.app_context.push()
         self.client = self.app.test_client()

tests/test_subprocess_backend.py

@@ -1,7 +1,6 @@
 import unittest
 from unittest.mock import MagicMock, patch
 
-from cellxgene_gateway.backend_cache import BackendCache
 from cellxgene_gateway.cache_entry import CacheEntry
 from cellxgene_gateway.cache_key import CacheKey
 from cellxgene_gateway.items.file.fileitem import FileItem
@@ -33,10 +32,46 @@ class TestSubprocessBackend(unittest.TestCase):
             backend.launch(cellxgene_loc, scripts, entry)
         popen.assert_called_once_with(
             [
-                "yes | /some/cellxgene launch /tmp/czi/pbmc3k.h5ad --port 8000 --host 127.0.0.1 --disable-annotations --scripts http://example.com/script.js --scripts http://example.com/script2.js"
+                "yes | /some/cellxgene launch /tmp/czi/pbmc3k.h5ad --port 8000 --host 127.0.0.1 --disable-annotations --disable-gene-sets-save --scripts http://example.com/script.js --scripts http://example.com/script2.js"
             ],
             shell=True,
             stderr=-1,
             stdout=-1,
         )
         self.assertEqual("An unexpected error", context.exception.stderr)
+
+    @patch("subprocess.Popen")
+    def test_launch_GIVEN_annotations_enabled_THEN_set_flags(self, popen):
+        subprocess = MagicMock()
+        subprocess.stdout.readline().decode.return_value = (
+            "[cellxgene] Type CTRL-C at any time to exit.\n"
+        )
+        subprocess.stderr.read().decode.return_value = ""
+        popen.return_value = subprocess
+
+        key = CacheKey(
+            FileItem("/czi/", name="pbmc3k.h5ad", type=ItemType.h5ad),
+            FileItemSource("/tmp", "local"),
+            FileItem(
+                "/czi/pbmc3k_annotations/", name="foo.csv", type=ItemType.annotation
+            ),
+        )
+        entry = CacheEntry.for_key(key, 8000)
+        import cellxgene_gateway.subprocess_backend
+
+        cellxgene_gateway.subprocess_backend.enable_annotations = True
+        try:
+            backend = cellxgene_gateway.subprocess_backend.SubprocessBackend()
+            cellxgene_loc = "/some/cellxgene"
+
+            backend.launch(cellxgene_loc, [], entry)
+        finally:
+            cellxgene_gateway.subprocess_backend.enable_annotations = False
+        popen.assert_called_once_with(
+            [
+                "yes | /some/cellxgene launch /tmp/czi/pbmc3k.h5ad --port 8000 --host 127.0.0.1 --annotations-file /tmp/czi/pbmc3k_annotations/foo.csv --gene-sets-file /tmp/czi/pbmc3k_annotations/foo_gene_sets.csv"
+            ],
+            shell=True,
+            stderr=-1,
+            stdout=-1,
+        )