From 81c8ce4219788b5036d27be73f7a96b107449c54 Mon Sep 17 00:00:00 2001 From: george-hall-ucl Date: Wed, 5 Jul 2023 12:41:47 +0100 Subject: [PATCH] #81 Add support for gene sets This adds the flag `GATEWAY_ENABLE_GENE_SETS` to enable support for gene sets. To simplify implementation, activating this flag also activates `GATEWAY_ENABLE_ANNOTATIONS`. The gene sets are saved in a file that has the same name as the annotations `csv` but with `_gene_sets` appended to the file name (before the extension). This file is hidden in filecrawler, and the gene sets are loaded when the associated annotations file is loaded. If the annotations file is missing, then an Exception is raised. I have updated one unit test to make it expect `--disable-gene-sets-save` in the default case (i.e. if `GATEWAY_ENABLE_ANNOTATIONS = 0`). All units tests pass. I have updated the README to document `GATEWAY_ENABLE_GENE_SETS`. --- README.md | 1 + cellxgene_gateway/env.py | 7 +++++++ cellxgene_gateway/filecrawl.py | 12 ++++++++++-- cellxgene_gateway/subprocess_backend.py | 17 ++++++++++++++++- tests/test_subprocess_backend.py | 2 +- 5 files changed, 35 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3200042..eed75f6 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,7 @@ Optional environment variables: * `GATEWAY_EXPIRE_SECONDS` - time in seconds that a cellxgene process will remain idle before being terminated. Defaults to 3600 (one hour) * `GATEWAY_EXTRA_SCRIPTS` - JSON array of script paths, will be embedded into each page and forwarded with `--scripts` to cellxgene server * `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations. +* `GATEWAY_ENABLE_GENE_SETS` - Set to `true` or to `1` to enable cellxgene gene sets. Also enables `GATEWAY_ENABLE_ANNOTATIONS`. * `GATEWAY_ENABLE_BACKED_MODE` - Set to `true` or to `1` to load AnnData in file-backed mode. This saves memory and speeds up launch time but may reduce overall performance. * `GATEWAY_LOG_LEVEL` - default is `INFO`. set to `DEBUG` to increase logging and to `WARNING` to decrease logging. * `S3_ENABLE_LISTINGS_CACHE` - Set to `true` or to `1` to cache listings of S3 folders for performance. If the cache becomes stale, set `filecrawl.html?refresh=true` query parameter to refresh the cache. diff --git a/cellxgene_gateway/env.py b/cellxgene_gateway/env.py index 5cf0c1b..4d8c4db 100644 --- a/cellxgene_gateway/env.py +++ b/cellxgene_gateway/env.py @@ -26,10 +26,16 @@ extra_scripts = os.environ.get("GATEWAY_EXTRA_SCRIPTS") expire_seconds = int( os.environ.get("GATEWAY_EXPIRE_SECONDS", os.environ.get("GATEWAY_TTL", "3600")) ) +enable_gene_sets = os.environ.get("GATEWAY_ENABLE_GENE_SETS", "").lower() in [ + "true", + "1", +] enable_annotations = os.environ.get("GATEWAY_ENABLE_ANNOTATIONS", "").lower() in [ "true", "1", ] +# Enable annotations if gene sets are enabled: +enable_annotations = enable_annotations or enable_gene_sets enable_backed_mode = os.environ.get("GATEWAY_ENABLE_BACKED_MODE", "").lower() in [ "true", "1", @@ -54,6 +60,7 @@ optional_env_vars = { "GATEWAY_EXTRA_SCRIPTS": extra_scripts, "GATEWAY_EXPIRE_SECONDS": expire_seconds, "GATEWAY_ENABLE_ANNOTATIONS": enable_annotations, + "GATEWAY_ENABLE_GENE_SETS": enable_gene_sets, "GATEWAY_ENABLE_BACKED_MODE": enable_backed_mode, "GATEWAY_LOG_LEVEL": log_level, "CELLXGENE_ARGS": cellxgene_args, diff --git a/cellxgene_gateway/filecrawl.py b/cellxgene_gateway/filecrawl.py index f812091..582afe8 100644 --- a/cellxgene_gateway/filecrawl.py +++ b/cellxgene_gateway/filecrawl.py @@ -20,15 +20,23 @@ def render_annotations(item, item_source): item_source.get_annotations_subpath(item), item_source.name ) new_annotation = f"new" + non_gene_set_files = [] + if item.annotations is not None: + # Do not also display files to store gene_sets. These should be loaded + # by clicking on the associated annotations file (i.e. without the + # appended "_gene_sets") + for a in item.annotations: + if (len(a.name) < 10) or (a.name[-10:] != "_gene_sets"): + non_gene_set_files.append(a) annotations = ( ", ".join( [ f"{a.name}" - for a in item.annotations + for a in non_gene_set_files ] ) + ", " - if item.annotations + if non_gene_set_files else "" ) return " | annotations: " + annotations + new_annotation diff --git a/cellxgene_gateway/subprocess_backend.py b/cellxgene_gateway/subprocess_backend.py index 19f8dc9..8825875 100644 --- a/cellxgene_gateway/subprocess_backend.py +++ b/cellxgene_gateway/subprocess_backend.py @@ -14,7 +14,12 @@ from flask_api import status from cellxgene_gateway.cache_entry import CacheEntryStatus from cellxgene_gateway.dir_util import make_annotations -from cellxgene_gateway.env import cellxgene_args, enable_annotations, enable_backed_mode +from cellxgene_gateway.env import ( + cellxgene_args, + enable_annotations, + enable_backed_mode, + enable_gene_sets, +) from cellxgene_gateway.process_exception import ProcessException logger = logging.getLogger(__name__) @@ -32,6 +37,16 @@ class SubprocessBackend: extra_args = f" --annotations-file {annotation_file_path}" else: extra_args = " --disable-annotations" + if enable_gene_sets and not annotation_file_path is None: + if annotation_file_path == "": + raise Exception( + "GATEWAY_ENABLE_GENE_SETS is true but --annotation_file_path not set" + ) + else: + gene_sets_file_path = annotation_file_path[:-4] + "_gene_sets.csv" + extra_args += f" --gene-sets-file {gene_sets_file_path}" + else: + extra_args += " --disable-gene-sets-save" if enable_backed_mode: extra_args += " --backed" if not cellxgene_args is None: diff --git a/tests/test_subprocess_backend.py b/tests/test_subprocess_backend.py index 5c22dbf..62652bf 100644 --- a/tests/test_subprocess_backend.py +++ b/tests/test_subprocess_backend.py @@ -33,7 +33,7 @@ class TestSubprocessBackend(unittest.TestCase): backend.launch(cellxgene_loc, scripts, entry) popen.assert_called_once_with( [ - "yes | /some/cellxgene launch /tmp/czi/pbmc3k.h5ad --port 8000 --host 127.0.0.1 --disable-annotations --scripts http://example.com/script.js --scripts http://example.com/script2.js" + "yes | /some/cellxgene launch /tmp/czi/pbmc3k.h5ad --port 8000 --host 127.0.0.1 --disable-annotations --disable-gene-sets-save --scripts http://example.com/script.js --scripts http://example.com/script2.js" ], shell=True, stderr=-1,