#81 Add support for gene sets

This adds the flag `GATEWAY_ENABLE_GENE_SETS` to enable support for gene
sets.  To simplify implementation, activating this flag also activates
`GATEWAY_ENABLE_ANNOTATIONS`.  The gene sets are saved in a file that
has the same name as the annotations `csv` but with `_gene_sets`
appended to the file name (before the extension).  This file is hidden
in filecrawler, and the gene sets are loaded when the associated
annotations file is loaded.

If the annotations file is missing, then an Exception is raised.

I have updated one unit test to make it expect
`--disable-gene-sets-save` in the default case (i.e. if
`GATEWAY_ENABLE_ANNOTATIONS = 0`).  All units tests pass.

I have updated the README to document `GATEWAY_ENABLE_GENE_SETS`.

README.md

@@ -76,6 +76,7 @@ Optional environment variables:
 * `GATEWAY_EXPIRE_SECONDS` - time in seconds that a cellxgene process will remain idle before being terminated. Defaults to 3600 (one hour)
 * `GATEWAY_EXTRA_SCRIPTS` - JSON array of script paths, will be embedded into each page and forwarded with `--scripts` to cellxgene server
 * `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations. 
+* `GATEWAY_ENABLE_GENE_SETS` - Set to `true` or to `1` to enable cellxgene gene sets. Also enables `GATEWAY_ENABLE_ANNOTATIONS`.
 * `GATEWAY_ENABLE_BACKED_MODE` - Set to `true` or to `1` to load AnnData in file-backed mode. This saves memory and speeds up launch time but may reduce overall performance.
 * `GATEWAY_LOG_LEVEL` - default is `INFO`. set to `DEBUG` to increase logging and to `WARNING` to decrease logging.
 * `S3_ENABLE_LISTINGS_CACHE` - Set to `true` or to `1` to cache listings of S3 folders for performance. If the cache becomes stale, set `filecrawl.html?refresh=true` query parameter to refresh the cache.

cellxgene_gateway/env.py

@@ -26,10 +26,16 @@ extra_scripts = os.environ.get("GATEWAY_EXTRA_SCRIPTS")
 expire_seconds = int(
     os.environ.get("GATEWAY_EXPIRE_SECONDS", os.environ.get("GATEWAY_TTL", "3600"))
 )
+enable_gene_sets = os.environ.get("GATEWAY_ENABLE_GENE_SETS", "").lower() in [
+    "true",
+    "1",
+]
 enable_annotations = os.environ.get("GATEWAY_ENABLE_ANNOTATIONS", "").lower() in [
     "true",
     "1",
 ]
+# Enable annotations if gene sets are enabled:
+enable_annotations = enable_annotations or enable_gene_sets
 enable_backed_mode = os.environ.get("GATEWAY_ENABLE_BACKED_MODE", "").lower() in [
     "true",
     "1",
@@ -54,6 +60,7 @@ optional_env_vars = {
     "GATEWAY_EXTRA_SCRIPTS": extra_scripts,
     "GATEWAY_EXPIRE_SECONDS": expire_seconds,
     "GATEWAY_ENABLE_ANNOTATIONS": enable_annotations,
+    "GATEWAY_ENABLE_GENE_SETS": enable_gene_sets,
     "GATEWAY_ENABLE_BACKED_MODE": enable_backed_mode,
     "GATEWAY_LOG_LEVEL": log_level,
     "CELLXGENE_ARGS": cellxgene_args,

cellxgene_gateway/filecrawl.py

@@ -20,15 +20,23 @@ def render_annotations(item, item_source):
         item_source.get_annotations_subpath(item), item_source.name
     )
     new_annotation = f"<a class='new' href='{url}'>new</a>"
+    non_gene_set_files = []
+    if item.annotations is not None:
+        # Do not also display files to store gene_sets. These should be loaded
+        # by clicking on the associated annotations file (i.e.  without the
+        # appended "_gene_sets")
+        for a in item.annotations:
+            if (len(a.name) < 10) or (a.name[-10:] != "_gene_sets"):
+                non_gene_set_files.append(a)
     annotations = (
         ", ".join(
             [
                 f"<a href='{CacheKey(item, item_source, a).view_url}/'>{a.name}</a>"
-                for a in item.annotations
+                for a in non_gene_set_files
             ]
         )
         + ", "
-        if item.annotations
+        if non_gene_set_files
         else ""
     )
     return " | annotations: " + annotations + new_annotation

cellxgene_gateway/subprocess_backend.py

@@ -14,7 +14,12 @@ from flask_api import status
 
 from cellxgene_gateway.cache_entry import CacheEntryStatus
 from cellxgene_gateway.dir_util import make_annotations
-from cellxgene_gateway.env import cellxgene_args, enable_annotations, enable_backed_mode
+from cellxgene_gateway.env import (
+    cellxgene_args,
+    enable_annotations,
+    enable_backed_mode,
+    enable_gene_sets,
+)
 from cellxgene_gateway.process_exception import ProcessException
 
 logger = logging.getLogger(__name__)
@@ -32,6 +37,16 @@ class SubprocessBackend:
                 extra_args = f" --annotations-file {annotation_file_path}"
         else:
             extra_args = " --disable-annotations"
+        if enable_gene_sets and not annotation_file_path is None:
+            if annotation_file_path == "":
+                raise Exception(
+                    "GATEWAY_ENABLE_GENE_SETS is true but --annotation_file_path not set"
+                )
+            else:
+                gene_sets_file_path = annotation_file_path[:-4] + "_gene_sets.csv"
+                extra_args += f" --gene-sets-file {gene_sets_file_path}"
+        else:
+            extra_args += " --disable-gene-sets-save"
         if enable_backed_mode:
             extra_args += " --backed"
         if not cellxgene_args is None:

tests/test_subprocess_backend.py

@@ -33,7 +33,7 @@ class TestSubprocessBackend(unittest.TestCase):
             backend.launch(cellxgene_loc, scripts, entry)
         popen.assert_called_once_with(
             [
-                "yes | /some/cellxgene launch /tmp/czi/pbmc3k.h5ad --port 8000 --host 127.0.0.1 --disable-annotations --scripts http://example.com/script.js --scripts http://example.com/script2.js"
+                "yes | /some/cellxgene launch /tmp/czi/pbmc3k.h5ad --port 8000 --host 127.0.0.1 --disable-annotations --disable-gene-sets-save --scripts http://example.com/script.js --scripts http://example.com/script2.js"
             ],
             shell=True,
             stderr=-1,