Merge pull request #83 from Novartis/81_gene_set_support

gene set support

.github/workflows/pr-checks.yaml

@@ -6,7 +6,7 @@ on: [push, pull_request]
 
 jobs:
   black:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
         name: Checkout repository
@@ -25,7 +25,7 @@ jobs:
           black . --check
   # This job is copied over from `deploy.yaml`
   run-tests:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
 

Changelog.md

@@ -1,3 +1,7 @@
+# 0.3.11
+
+* #81 added support for gene sets
+
 # 0.3.10
 
 * #65 Added GATEWAY_EXPIRE_SECONDS to set how long cellxgene servers can remain idle before being terminated.

README.md

@@ -75,7 +75,7 @@ Optional environment variables:
 * `GATEWAY_PORT` - local port that the gateway should bind to, defaults to 5005
 * `GATEWAY_EXPIRE_SECONDS` - time in seconds that a cellxgene process will remain idle before being terminated. Defaults to 3600 (one hour)
 * `GATEWAY_EXTRA_SCRIPTS` - JSON array of script paths, will be embedded into each page and forwarded with `--scripts` to cellxgene server
-* `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations. 
+* `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations and gene sets.
 * `GATEWAY_ENABLE_BACKED_MODE` - Set to `true` or to `1` to load AnnData in file-backed mode. This saves memory and speeds up launch time but may reduce overall performance.
 * `GATEWAY_LOG_LEVEL` - default is `INFO`. set to `DEBUG` to increase logging and to `WARNING` to decrease logging.
 * `S3_ENABLE_LISTINGS_CACHE` - Set to `true` or to `1` to cache listings of S3 folders for performance. If the cache becomes stale, set `filecrawl.html?refresh=true` query parameter to refresh the cache.

cellxgene_gateway/cache_entry.py

@@ -58,7 +58,6 @@ class CacheEntry:
 
     @classmethod
     def for_key(cls, key, port):
-
         return cls(
             None,
             key,

cellxgene_gateway/gateway.py

@@ -80,7 +80,6 @@ cache = BackendCache()
 
 @app.errorhandler(CellxgeneException)
 def handle_invalid_usage(error):
-
     message = f"{error.http_status} Error : {error.message}"
 
     return (
@@ -95,7 +94,6 @@ def handle_invalid_usage(error):
 
 @app.errorhandler(ProcessException)
 def handle_invalid_process(error):
-
     message = []
 
     message.append(error.message)

cellxgene_gateway/items/file/fileitem_source.py

@@ -35,6 +35,11 @@ class FileItemSource(ItemSource):
     def name(self):
         return self._name or f"Files:{self.base_path}"
 
+    def is_gene_set(self, path: str) -> bool:
+        return ("_gene_sets" in path or "-gene-sets" in path) and path.endswith(
+            self.annotation_file_suffix
+        )
+
     def is_h5ad_file(self, path: str) -> bool:
         return path.endswith(self.h5ad_suffix) and os.path.isfile(path)
 
@@ -180,6 +185,7 @@ class FileItemSource(ItemSource):
                 self.make_fileitem_from_path(annotation, annotations_subpath, True)
                 for annotation in sorted(os.listdir(annotations_fullpath))
                 if annotation.endswith(self.annotation_file_suffix)
+                and not self.is_gene_set(annotation)
                 and os.path.isfile(os.path.join(annotations_fullpath, annotation))
             ]
         else:

cellxgene_gateway/subprocess_backend.py

@@ -14,7 +14,11 @@ from flask_api import status
 
 from cellxgene_gateway.cache_entry import CacheEntryStatus
 from cellxgene_gateway.dir_util import make_annotations
-from cellxgene_gateway.env import cellxgene_args, enable_annotations, enable_backed_mode
+from cellxgene_gateway.env import (
+    cellxgene_args,
+    enable_annotations,
+    enable_backed_mode,
+)
 from cellxgene_gateway.process_exception import ProcessException
 
 logger = logging.getLogger(__name__)
@@ -30,8 +34,11 @@ class SubprocessBackend:
                 extra_args = f" --annotations-dir {make_annotations(file_path)}"
             else:
                 extra_args = f" --annotations-file {annotation_file_path}"
+                gene_sets_file_path = annotation_file_path[:-4] + "_gene_sets.csv"
+                extra_args += f" --gene-sets-file {gene_sets_file_path}"
         else:
             extra_args = " --disable-annotations"
+            extra_args += " --disable-gene-sets-save"
         if enable_backed_mode:
             extra_args += " --backed"
         if not cellxgene_args is None:

tests/test_subprocess_backend.py

@@ -1,7 +1,6 @@
 import unittest
 from unittest.mock import MagicMock, patch
 
-from cellxgene_gateway.backend_cache import BackendCache
 from cellxgene_gateway.cache_entry import CacheEntry
 from cellxgene_gateway.cache_key import CacheKey
 from cellxgene_gateway.items.file.fileitem import FileItem
@@ -33,10 +32,46 @@ class TestSubprocessBackend(unittest.TestCase):
             backend.launch(cellxgene_loc, scripts, entry)
         popen.assert_called_once_with(
             [
-                "yes | /some/cellxgene launch /tmp/czi/pbmc3k.h5ad --port 8000 --host 127.0.0.1 --disable-annotations --scripts http://example.com/script.js --scripts http://example.com/script2.js"
+                "yes | /some/cellxgene launch /tmp/czi/pbmc3k.h5ad --port 8000 --host 127.0.0.1 --disable-annotations --disable-gene-sets-save --scripts http://example.com/script.js --scripts http://example.com/script2.js"
             ],
             shell=True,
             stderr=-1,
             stdout=-1,
         )
         self.assertEqual("An unexpected error", context.exception.stderr)
+
+    @patch("subprocess.Popen")
+    def test_launch_GIVEN_annotations_enabled_THEN_set_flags(self, popen):
+        subprocess = MagicMock()
+        subprocess.stdout.readline().decode.return_value = (
+            "[cellxgene] Type CTRL-C at any time to exit.\n"
+        )
+        subprocess.stderr.read().decode.return_value = ""
+        popen.return_value = subprocess
+
+        key = CacheKey(
+            FileItem("/czi/", name="pbmc3k.h5ad", type=ItemType.h5ad),
+            FileItemSource("/tmp", "local"),
+            FileItem(
+                "/czi/pbmc3k_annotations/", name="foo.csv", type=ItemType.annotation
+            ),
+        )
+        entry = CacheEntry.for_key(key, 8000)
+        import cellxgene_gateway.subprocess_backend
+
+        cellxgene_gateway.subprocess_backend.enable_annotations = True
+        try:
+            backend = cellxgene_gateway.subprocess_backend.SubprocessBackend()
+            cellxgene_loc = "/some/cellxgene"
+
+            backend.launch(cellxgene_loc, [], entry)
+        finally:
+            cellxgene_gateway.subprocess_backend.enable_annotations = False
+        popen.assert_called_once_with(
+            [
+                "yes | /some/cellxgene launch /tmp/czi/pbmc3k.h5ad --port 8000 --host 127.0.0.1 --annotations-file /tmp/czi/pbmc3k_annotations/foo.csv --gene-sets-file /tmp/czi/pbmc3k_annotations/foo_gene_sets.csv"
+            ],
+            shell=True,
+            stderr=-1,
+            stdout=-1,
+        )