diff --git a/.github/workflows/pr-checks.yaml b/.github/workflows/pr-checks.yaml index 931d8f4..17d6cd0 100644 --- a/.github/workflows/pr-checks.yaml +++ b/.github/workflows/pr-checks.yaml @@ -6,7 +6,7 @@ on: [push, pull_request] jobs: black: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 name: Checkout repository @@ -25,7 +25,7 @@ jobs: black . --check # This job is copied over from `deploy.yaml` run-tests: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 diff --git a/Changelog.md b/Changelog.md index 2a93964..10aad9b 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,7 @@ +# 0.3.11 + +* #81 added support for gene sets + # 0.3.10 * #65 Added GATEWAY_EXPIRE_SECONDS to set how long cellxgene servers can remain idle before being terminated. diff --git a/README.md b/README.md index 3200042..13ba916 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ Optional environment variables: * `GATEWAY_PORT` - local port that the gateway should bind to, defaults to 5005 * `GATEWAY_EXPIRE_SECONDS` - time in seconds that a cellxgene process will remain idle before being terminated. Defaults to 3600 (one hour) * `GATEWAY_EXTRA_SCRIPTS` - JSON array of script paths, will be embedded into each page and forwarded with `--scripts` to cellxgene server -* `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations. +* `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations and gene sets. * `GATEWAY_ENABLE_BACKED_MODE` - Set to `true` or to `1` to load AnnData in file-backed mode. This saves memory and speeds up launch time but may reduce overall performance. * `GATEWAY_LOG_LEVEL` - default is `INFO`. set to `DEBUG` to increase logging and to `WARNING` to decrease logging. * `S3_ENABLE_LISTINGS_CACHE` - Set to `true` or to `1` to cache listings of S3 folders for performance. If the cache becomes stale, set `filecrawl.html?refresh=true` query parameter to refresh the cache. diff --git a/cellxgene_gateway/cache_entry.py b/cellxgene_gateway/cache_entry.py index aa3bb0c..8c72efc 100644 --- a/cellxgene_gateway/cache_entry.py +++ b/cellxgene_gateway/cache_entry.py @@ -58,7 +58,6 @@ class CacheEntry: @classmethod def for_key(cls, key, port): - return cls( None, key, diff --git a/cellxgene_gateway/gateway.py b/cellxgene_gateway/gateway.py index c930974..1b9c14e 100644 --- a/cellxgene_gateway/gateway.py +++ b/cellxgene_gateway/gateway.py @@ -80,7 +80,6 @@ cache = BackendCache() @app.errorhandler(CellxgeneException) def handle_invalid_usage(error): - message = f"{error.http_status} Error : {error.message}" return ( @@ -95,7 +94,6 @@ def handle_invalid_usage(error): @app.errorhandler(ProcessException) def handle_invalid_process(error): - message = [] message.append(error.message) diff --git a/cellxgene_gateway/items/file/fileitem_source.py b/cellxgene_gateway/items/file/fileitem_source.py index 4ade4b9..407f3a8 100644 --- a/cellxgene_gateway/items/file/fileitem_source.py +++ b/cellxgene_gateway/items/file/fileitem_source.py @@ -35,6 +35,11 @@ class FileItemSource(ItemSource): def name(self): return self._name or f"Files:{self.base_path}" + def is_gene_set(self, path: str) -> bool: + return ("_gene_sets" in path or "-gene-sets" in path) and path.endswith( + self.annotation_file_suffix + ) + def is_h5ad_file(self, path: str) -> bool: return path.endswith(self.h5ad_suffix) and os.path.isfile(path) @@ -180,6 +185,7 @@ class FileItemSource(ItemSource): self.make_fileitem_from_path(annotation, annotations_subpath, True) for annotation in sorted(os.listdir(annotations_fullpath)) if annotation.endswith(self.annotation_file_suffix) + and not self.is_gene_set(annotation) and os.path.isfile(os.path.join(annotations_fullpath, annotation)) ] else: diff --git a/cellxgene_gateway/subprocess_backend.py b/cellxgene_gateway/subprocess_backend.py index 19f8dc9..0dc87be 100644 --- a/cellxgene_gateway/subprocess_backend.py +++ b/cellxgene_gateway/subprocess_backend.py @@ -14,7 +14,11 @@ from flask_api import status from cellxgene_gateway.cache_entry import CacheEntryStatus from cellxgene_gateway.dir_util import make_annotations -from cellxgene_gateway.env import cellxgene_args, enable_annotations, enable_backed_mode +from cellxgene_gateway.env import ( + cellxgene_args, + enable_annotations, + enable_backed_mode, +) from cellxgene_gateway.process_exception import ProcessException logger = logging.getLogger(__name__) @@ -30,8 +34,11 @@ class SubprocessBackend: extra_args = f" --annotations-dir {make_annotations(file_path)}" else: extra_args = f" --annotations-file {annotation_file_path}" + gene_sets_file_path = annotation_file_path[:-4] + "_gene_sets.csv" + extra_args += f" --gene-sets-file {gene_sets_file_path}" else: extra_args = " --disable-annotations" + extra_args += " --disable-gene-sets-save" if enable_backed_mode: extra_args += " --backed" if not cellxgene_args is None: diff --git a/tests/test_subprocess_backend.py b/tests/test_subprocess_backend.py index 5c22dbf..f4f839e 100644 --- a/tests/test_subprocess_backend.py +++ b/tests/test_subprocess_backend.py @@ -1,7 +1,6 @@ import unittest from unittest.mock import MagicMock, patch -from cellxgene_gateway.backend_cache import BackendCache from cellxgene_gateway.cache_entry import CacheEntry from cellxgene_gateway.cache_key import CacheKey from cellxgene_gateway.items.file.fileitem import FileItem @@ -33,10 +32,46 @@ class TestSubprocessBackend(unittest.TestCase): backend.launch(cellxgene_loc, scripts, entry) popen.assert_called_once_with( [ - "yes | /some/cellxgene launch /tmp/czi/pbmc3k.h5ad --port 8000 --host 127.0.0.1 --disable-annotations --scripts http://example.com/script.js --scripts http://example.com/script2.js" + "yes | /some/cellxgene launch /tmp/czi/pbmc3k.h5ad --port 8000 --host 127.0.0.1 --disable-annotations --disable-gene-sets-save --scripts http://example.com/script.js --scripts http://example.com/script2.js" ], shell=True, stderr=-1, stdout=-1, ) self.assertEqual("An unexpected error", context.exception.stderr) + + @patch("subprocess.Popen") + def test_launch_GIVEN_annotations_enabled_THEN_set_flags(self, popen): + subprocess = MagicMock() + subprocess.stdout.readline().decode.return_value = ( + "[cellxgene] Type CTRL-C at any time to exit.\n" + ) + subprocess.stderr.read().decode.return_value = "" + popen.return_value = subprocess + + key = CacheKey( + FileItem("/czi/", name="pbmc3k.h5ad", type=ItemType.h5ad), + FileItemSource("/tmp", "local"), + FileItem( + "/czi/pbmc3k_annotations/", name="foo.csv", type=ItemType.annotation + ), + ) + entry = CacheEntry.for_key(key, 8000) + import cellxgene_gateway.subprocess_backend + + cellxgene_gateway.subprocess_backend.enable_annotations = True + try: + backend = cellxgene_gateway.subprocess_backend.SubprocessBackend() + cellxgene_loc = "/some/cellxgene" + + backend.launch(cellxgene_loc, [], entry) + finally: + cellxgene_gateway.subprocess_backend.enable_annotations = False + popen.assert_called_once_with( + [ + "yes | /some/cellxgene launch /tmp/czi/pbmc3k.h5ad --port 8000 --host 127.0.0.1 --annotations-file /tmp/czi/pbmc3k_annotations/foo.csv --gene-sets-file /tmp/czi/pbmc3k_annotations/foo_gene_sets.csv" + ], + shell=True, + stderr=-1, + stdout=-1, + )