#81 added unit test for gene sets

This adds the flag `GATEWAY_ENABLE_GENE_SETS` to enable support for gene
sets.  To simplify implementation, activating this flag also activates
`GATEWAY_ENABLE_ANNOTATIONS`.  The gene sets are saved in a file that
has the same name as the annotations `csv` but with `_gene_sets`
appended to the file name (before the extension).  This file is hidden
in filecrawler, and the gene sets are loaded when the associated
annotations file is loaded.

If the annotations file is missing, then an Exception is raised.

I have updated one unit test to make it expect
`--disable-gene-sets-save` in the default case (i.e. if
`GATEWAY_ENABLE_ANNOTATIONS = 0`).  All units tests pass.

I have updated the README to document `GATEWAY_ENABLE_GENE_SETS`.

.github/workflows/pr-checks.yaml

@@ -6,7 +6,7 @@ on: [push, pull_request]
 
 jobs:
   black:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
         name: Checkout repository
@@ -25,7 +25,7 @@ jobs:
           black . --check
   # This job is copied over from `deploy.yaml`
   run-tests:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
 

Changelog.md

@@ -1,3 +1,7 @@
+# 0.3.11
+
+* #81 added support for gene sets
+
 # 0.3.10
 
 * #65 Added GATEWAY_EXPIRE_SECONDS to set how long cellxgene servers can remain idle before being terminated.

README.md

@@ -75,7 +75,7 @@ Optional environment variables:
 * `GATEWAY_PORT` - local port that the gateway should bind to, defaults to 5005
 * `GATEWAY_EXPIRE_SECONDS` - time in seconds that a cellxgene process will remain idle before being terminated. Defaults to 3600 (one hour)
 * `GATEWAY_EXTRA_SCRIPTS` - JSON array of script paths, will be embedded into each page and forwarded with `--scripts` to cellxgene server
-* `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations. 
+* `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations and gene sets.
 * `GATEWAY_ENABLE_BACKED_MODE` - Set to `true` or to `1` to load AnnData in file-backed mode. This saves memory and speeds up launch time but may reduce overall performance.
 * `GATEWAY_LOG_LEVEL` - default is `INFO`. set to `DEBUG` to increase logging and to `WARNING` to decrease logging.
 * `S3_ENABLE_LISTINGS_CACHE` - Set to `true` or to `1` to cache listings of S3 folders for performance. If the cache becomes stale, set `filecrawl.html?refresh=true` query parameter to refresh the cache.

cellxgene_gateway/cache_entry.py

@@ -58,7 +58,6 @@ class CacheEntry:
 
     @classmethod
     def for_key(cls, key, port):
-
         return cls(
             None,
             key,

cellxgene_gateway/filecrawl.py

@@ -7,7 +7,7 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
-import html
+import os
 import urllib.parse
 
 from cellxgene_gateway import env, flask_util
@@ -19,17 +19,19 @@ def render_annotations(item, item_source):
     url = flask_util.view_url(
         item_source.get_annotations_subpath(item), item_source.name
     )
-    new_annotation = [f"<a class='new' href='{url}'>new</a>"]
-
+    new_annotation = f"<a class='new' href='{url}'>new</a>"
     annotations = (
-        [
-            f"<a href='{CacheKey(item, item_source, a).view_url}/'>{html.escape(a.name)}</a>"
-            for a in item.annotations
-        ]
+        ", ".join(
+            [
+                f"<a href='{CacheKey(item, item_source, a).view_url}/'>{a.name}</a>"
+                for a in item.annotations
+            ]
+        )
+        + ", "
         if item.annotations
-        else []
+        else ""
     )
-    return "| annotations: " + ", ".join(new_annotation + annotations)
+    return " | annotations: " + annotations + new_annotation
 
 
 def render_item(item, item_source):

cellxgene_gateway/gateway.py

@@ -80,7 +80,6 @@ cache = BackendCache()
 
 @app.errorhandler(CellxgeneException)
 def handle_invalid_usage(error):
-
     message = f"{error.http_status} Error : {error.message}"
 
     return (
@@ -95,7 +94,6 @@ def handle_invalid_usage(error):
 
 @app.errorhandler(ProcessException)
 def handle_invalid_process(error):
-
     message = []
 
     message.append(error.message)

cellxgene_gateway/items/file/fileitem_source.py

@@ -35,6 +35,11 @@ class FileItemSource(ItemSource):
     def name(self):
         return self._name or f"Files:{self.base_path}"
 
+    def is_gene_set(self, path: str) -> bool:
+        return ("_gene_sets" in path or "-gene-sets" in path) and path.endswith(
+            self.annotation_file_suffix
+        )
+
     def is_h5ad_file(self, path: str) -> bool:
         return path.endswith(self.h5ad_suffix) and os.path.isfile(path)
 
@@ -180,6 +185,7 @@ class FileItemSource(ItemSource):
                 self.make_fileitem_from_path(annotation, annotations_subpath, True)
                 for annotation in sorted(os.listdir(annotations_fullpath))
                 if annotation.endswith(self.annotation_file_suffix)
+                and not self.is_gene_set(annotation)
                 and os.path.isfile(os.path.join(annotations_fullpath, annotation))
             ]
         else:

cellxgene_gateway/subprocess_backend.py

@@ -14,7 +14,11 @@ from flask_api import status
 
 from cellxgene_gateway.cache_entry import CacheEntryStatus
 from cellxgene_gateway.dir_util import make_annotations
-from cellxgene_gateway.env import cellxgene_args, enable_annotations, enable_backed_mode
+from cellxgene_gateway.env import (
+    cellxgene_args,
+    enable_annotations,
+    enable_backed_mode,
+)
 from cellxgene_gateway.process_exception import ProcessException
 
 logger = logging.getLogger(__name__)
@@ -30,8 +34,11 @@ class SubprocessBackend:
                 extra_args = f" --annotations-dir {make_annotations(file_path)}"
             else:
                 extra_args = f" --annotations-file {annotation_file_path}"
+                gene_sets_file_path = annotation_file_path[:-4] + "_gene_sets.csv"
+                extra_args += f" --gene-sets-file {gene_sets_file_path}"
         else:
             extra_args = " --disable-annotations"
+            extra_args += " --disable-gene-sets-save"
         if enable_backed_mode:
             extra_args += " --backed"
         if not cellxgene_args is None:

tests/test_filecrawl.py

@@ -13,76 +13,26 @@ from cellxgene_gateway.items.item import ItemTree, ItemType
 source = FileItemSource("/tmp")
 
 
-def make_entry(subpath="somepath", annotations=None):
-    return FileItem(
-        subpath=subpath,
-        name="entry",
-        ext=".h5ad",
-        type=ItemType.h5ad,
-        annotations=annotations,
-    )
-
-
 class TestRenderEntry(unittest.TestCase):
     def test_GIVEN_path_both_slash_THEN_view_has_single_slash(self):
-        entry = make_entry(subpath="/somepath/")
+        entry = FileItem(subpath="/somepath/", name="entry", type=ItemType.h5ad)
         rendered = render_item(entry, source)
-        self.assertIn("view/somepath/entry.h5ad/'", rendered)
+        self.assertIn("view/somepath/entry/'", rendered)
 
     def test_GIVEN_path_starts_slash_THEN_view_has_single_slash(self):
-        entry = make_entry(subpath="/somepath")
+        entry = FileItem(subpath="/somepath", name="entry", type=ItemType.h5ad)
         rendered = render_item(entry, source)
-        self.assertIn("view/somepath/entry.h5ad/'", rendered)
+        self.assertIn("view/somepath/entry/'", rendered)
 
     def test_GIVEN_path_ends_slash_THEN_view_has_single_slash(self):
-        entry = make_entry(subpath="somepath/")
+        entry = FileItem(subpath="somepath/", name="entry", type=ItemType.h5ad)
         rendered = render_item(entry, source)
-        self.assertIn("view/somepath/entry.h5ad/'", rendered)
+        self.assertIn("view/somepath/entry/'", rendered)
 
     def test_GIVEN_path_no_slash_THEN_view_has_single_slash(self):
-        entry = make_entry(subpath="somepath")
+        entry = FileItem(subpath="somepath", name="entry", type=ItemType.h5ad)
         rendered = render_item(entry, source)
-        self.assertIn("view/somepath/entry.h5ad/'", rendered)
-
-
-class TestRenderAnnotation(unittest.TestCase):
-    def test_GIVEN_no_annotation_THEN_new_alone(self):
-        entry = make_entry(annotations=None)
-        rendered = render_item(entry, source)
-        self.assertIn(
-            "> | annotations: <a class='new' href='/source/Files:/tmp/view/somepath/entry_annotations'>new</a></li>",
-            rendered,
-        )
-
-    def test_GIVEN_annotation_THEN_new_before(self):
-        annotation = FileItem(
-            subpath="somepath/entry_annotations",
-            name="annot",
-            ext=".csv",
-            type=ItemType.annotation,
-        )
-        entry = make_entry(annotations=[annotation])
-        rendered = render_item(entry, source)
-        self.assertIn(
-            "> | annotations: <a class='new' href='/source/Files:/tmp/view/somepath/entry_annotations'>new</a>,"
-            " <a href='/source/Files:/tmp/view/somepath/entry_annotations/annot.csv/'>annot</a></li>",
-            rendered,
-        )
-
-    def test_GIVEN_annotation_THEN_escaped(self):
-        annotation = FileItem(
-            subpath="somepath/entry_annotations",
-            name="hot&cold",
-            ext=".csv",
-            type=ItemType.annotation,
-        )
-        entry = make_entry(annotations=[annotation])
-        rendered = render_item(entry, source)
-        self.assertIn(
-            "> | annotations: <a class='new' href='/source/Files:/tmp/view/somepath/entry_annotations'>new</a>,"
-            " <a href='/source/Files:/tmp/view/somepath/entry_annotations/hot%26cold.csv/'>hot&amp;cold</a></li>",
-            rendered,
-        )
+        self.assertIn("view/somepath/entry/'", rendered)
 
 
 class TestRenderItemSource(unittest.TestCase):

tests/test_subprocess_backend.py

@@ -1,7 +1,6 @@
 import unittest
 from unittest.mock import MagicMock, patch
 
-from cellxgene_gateway.backend_cache import BackendCache
 from cellxgene_gateway.cache_entry import CacheEntry
 from cellxgene_gateway.cache_key import CacheKey
 from cellxgene_gateway.items.file.fileitem import FileItem
@@ -33,10 +32,46 @@ class TestSubprocessBackend(unittest.TestCase):
             backend.launch(cellxgene_loc, scripts, entry)
         popen.assert_called_once_with(
             [
-                "yes | /some/cellxgene launch /tmp/czi/pbmc3k.h5ad --port 8000 --host 127.0.0.1 --disable-annotations --scripts http://example.com/script.js --scripts http://example.com/script2.js"
+                "yes | /some/cellxgene launch /tmp/czi/pbmc3k.h5ad --port 8000 --host 127.0.0.1 --disable-annotations --disable-gene-sets-save --scripts http://example.com/script.js --scripts http://example.com/script2.js"
             ],
             shell=True,
             stderr=-1,
             stdout=-1,
         )
         self.assertEqual("An unexpected error", context.exception.stderr)
+
+    @patch("subprocess.Popen")
+    def test_launch_GIVEN_annotations_enabled_THEN_set_flags(self, popen):
+        subprocess = MagicMock()
+        subprocess.stdout.readline().decode.return_value = (
+            "[cellxgene] Type CTRL-C at any time to exit.\n"
+        )
+        subprocess.stderr.read().decode.return_value = ""
+        popen.return_value = subprocess
+
+        key = CacheKey(
+            FileItem("/czi/", name="pbmc3k.h5ad", type=ItemType.h5ad),
+            FileItemSource("/tmp", "local"),
+            FileItem(
+                "/czi/pbmc3k_annotations/", name="foo.csv", type=ItemType.annotation
+            ),
+        )
+        entry = CacheEntry.for_key(key, 8000)
+        import cellxgene_gateway.subprocess_backend
+
+        cellxgene_gateway.subprocess_backend.enable_annotations = True
+        try:
+            backend = cellxgene_gateway.subprocess_backend.SubprocessBackend()
+            cellxgene_loc = "/some/cellxgene"
+
+            backend.launch(cellxgene_loc, [], entry)
+        finally:
+            cellxgene_gateway.subprocess_backend.enable_annotations = False
+        popen.assert_called_once_with(
+            [
+                "yes | /some/cellxgene launch /tmp/czi/pbmc3k.h5ad --port 8000 --host 127.0.0.1 --annotations-file /tmp/czi/pbmc3k_annotations/foo.csv --gene-sets-file /tmp/czi/pbmc3k_annotations/foo_gene_sets.csv"
+            ],
+            shell=True,
+            stderr=-1,
+            stdout=-1,
+        )