Prepare 0.4.0 release

also remove dependency on flask-api

.github/workflows/pr-checks.yaml

@@ -39,7 +39,6 @@ jobs:
           conda env create -f environment.yml
           eval "$(conda shell.bash hook)"
           conda activate cellxgene-gateway
-          pip install markupsafe==2.0.1 # temporary workaround for jinja2-2.11.3 calling soft_unicode in markupsafe
           python setup.py install
 
       - name: Run tests

Changelog.md

@@ -1,6 +1,21 @@
+# 0.4.0
+
+* Removed dependency on flask-api
+* Updated dependencies (python 3.11, numpy, unpinned flask, werkzeug)
+
+# 0.3.12
+
+* #81 List gene set annotations when cell annotations not present
+* #86 Upgrade pip within docker image
+* #73 Moved new link to front
+* #87 Temporarily pin versions of werkzeug and flask
+
+
 # 0.3.11
 
 * #81 added support for gene sets
+* #79 added example for cellxgene-gateway customized docker image
+* #78 prune directories that do not contain h5ad files
 
 # 0.3.10
 

Dockerfile

@@ -1,6 +1,7 @@
-FROM python:3.9
+FROM python:3.11
 
-RUN pip install cellxgene-gateway 'MarkupSafe<2.1'
+RUN pip install --upgrade pip
+RUN pip install "cellxgene-gateway>=0.4"
 
 ENV CELLXGENE_DATA=/cellxgene-data
 ENV CELLXGENE_LOCATION=/usr/local/bin/cellxgene

README.md

@@ -110,7 +110,7 @@ Additional environment variables can be provided with the `-e` parameter:
 
 ```bash
 docker run -it --rm \
--v <local_data_dir>:/cellxgene-data \
+-v ../cellxgene_data:/cellxgene-data \
 -e GATEWAY_PORT=8080 \
 -p 8080:8080 \
 cellxgene-gateway
@@ -191,6 +191,35 @@ black .
 
 If you need help for any reason, please make a github ticket. One of the contributors should help you out.
 
+# Releasing New Versions
+
+## How to prepare for release
+
+- Update Changelog.md and version number in __init__.py
+- Cut a release on github
+	- Go to your project homepage on GitHub
+	- On right side, you will see [Releases](https://github.com/Novartis/cellxgene-gateway/releases) link. Click on it.
+	- Click on Draft a new release
+	- Fill in all the details
+		- Tag version should be the version number of your package release
+		- Release Title can be anything you want, but we use v0.3.11 (the same as the tag to be created on publish)
+		- Description should be changelog
+	- Click Publish release at the bottom of the page
+	- Now under Releases you can view all of your releases.
+	- Copy the download link (tar.gz) and save it somewhere
+
+## How to publish to PyPI
+
+Make sure your `.pypirc` is set up for testpypi and pypi index servers.
+ 
+
+```bash
+rm -rf dist
+python setup.py sdist bdist_wheel
+python -m twine upload --repository testpypi dist/*
+python -m twine upload dist/*
+```
+
 # Contributors
 
 * Niket Patel - https://github.com/NiketPatel9

cellxgene_gateway/__init__.py

@@ -7,4 +7,4 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
-__version__ = "0.3.10"
+__version__ = "0.4.0"

cellxgene_gateway/backend_cache.py

@@ -8,11 +8,10 @@
 # the specific language governing permissions and limitations under the License.
 
 import time
+from http import HTTPStatus
 from threading import Thread
 from typing import List
 
-from flask_api import status
-
 from cellxgene_gateway import env
 from cellxgene_gateway.cache_entry import CacheEntry, CacheEntryStatus
 from cellxgene_gateway.cache_key import CacheKey
@@ -53,7 +52,7 @@ class BackendCache:
             return matches[0]
         else:
             raise CellxgeneException(
-                status.HTTP_500_INTERNAL_SERVER_ERROR,
+                HTTPStatus.INTERNAL_SERVER_ERROR,
                 "Found " + str(len(matches)) + " for " + path,
             )
 
@@ -71,7 +70,7 @@ class BackendCache:
             return matches[0]
         else:
             raise CellxgeneException(
-                status.HTTP_500_INTERNAL_SERVER_ERROR,
+                HTTPStatus.INTERNAL_SERVER_ERROR,
                 "Found " + str(len(matches)) + " for " + key.dataset,
             )
 

cellxgene_gateway/dir_util.py

@@ -9,8 +9,6 @@
 
 import os
 
-from flask_api import status
-
 from cellxgene_gateway import env
 from cellxgene_gateway.cellxgene_exception import CellxgeneException
 

cellxgene_gateway/filecrawl.py

@@ -7,31 +7,32 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
-import os
+import html
 import urllib.parse
 
-from cellxgene_gateway import env, flask_util
+from cellxgene_gateway import flask_util
 from cellxgene_gateway.cache_key import CacheKey
 from cellxgene_gateway.dir_util import annotations_suffix, make_annotations, make_h5ad
+from cellxgene_gateway.env import enable_annotations
 
 
 def render_annotations(item, item_source):
+    if not enable_annotations:
+        return ""
     url = flask_util.view_url(
         item_source.get_annotations_subpath(item), item_source.name
     )
-    new_annotation = f"<a class='new' href='{url}'>new</a>"
+    new_annotation = [f"<a class='new' href='{url}'>new</a>"]
+
     annotations = (
-        ", ".join(
+        [
-            [
+            f"<a href='{CacheKey(item, item_source, a).view_url}/'>{html.escape(a.name)}</a>"
-                f"<a href='{CacheKey(item, item_source, a).view_url}/'>{a.name}</a>"
+            for a in item.annotations
-                for a in item.annotations
+        ]
-            ]
-        )
-        + ", "
         if item.annotations
-        else ""
+        else []
     )
-    return " | annotations: " + annotations + new_annotation
+    return "| annotations: " + ", ".join(new_annotation + annotations)
 
 
 def render_item(item, item_source):

cellxgene_gateway/items/file/fileitem_source.py

@@ -24,21 +24,21 @@ class FileItemSource(ItemSource):
         h5ad_suffix=dir_util.h5ad_suffix,
         annotation_dir_suffix=dir_util.annotations_suffix,
         annotation_file_suffix=".csv",
+        gene_set_file_suffix="_gene_sets.csv",
     ):
         self._name = name
         self.base_path = base_path
         self.h5ad_suffix = h5ad_suffix
         self.annotation_dir_suffix = annotation_dir_suffix
         self.annotation_file_suffix = annotation_file_suffix
+        self.gene_set_file_suffix = gene_set_file_suffix
 
     @property
     def name(self):
         return self._name or f"Files:{self.base_path}"
 
     def is_gene_set(self, path: str) -> bool:
-        return ("_gene_sets" in path or "-gene-sets" in path) and path.endswith(
+        return path.endswith(self.gene_set_file_suffix)
-            self.annotation_file_suffix
-        )
 
     def is_h5ad_file(self, path: str) -> bool:
         return path.endswith(self.h5ad_suffix) and os.path.isfile(path)
@@ -68,7 +68,7 @@ class FileItemSource(ItemSource):
 
         return item_tree
 
-    def scan_directory(self, subpath="") -> dict:
+    def scan_directory(self, subpath: str = "") -> ItemTree:
         base_path = os.path.join(self.base_path, subpath)
 
         if not os.path.exists(base_path):
@@ -105,6 +105,11 @@ class FileItemSource(ItemSource):
             branches = [
                 self.scan_directory(os.path.join(subpath, subdir)) for subdir in subdirs
             ]
+            # Exclude branches without files as leaves. Since traversal is applied pre-order,
+            # branch.branches has already been processed and we don't need to check deeper nesting.
+            branches = [
+                branch for branch in branches if branch.items or branch.branches
+            ]
 
         return ItemTree(subpath, items, branches)
 
@@ -181,12 +186,29 @@ class FileItemSource(ItemSource):
         annotations_subpath = self.get_annotations_subpath(item)
         annotations_fullpath = self.full_path(annotations_subpath)
         if os.path.isdir(annotations_fullpath):
-            return [
+            sorted_files = sorted(os.listdir(annotations_fullpath))
+            annotation_files = [
                 self.make_fileitem_from_path(annotation, annotations_subpath, True)
-                for annotation in sorted(os.listdir(annotations_fullpath))
+                for annotation in sorted_files
                 if annotation.endswith(self.annotation_file_suffix)
                 and not self.is_gene_set(annotation)
                 and os.path.isfile(os.path.join(annotations_fullpath, annotation))
             ]
+
+            # Catch gene sets without accompanying [annotations].csv
+            gene_sets_files = [
+                self.make_fileitem_from_path(
+                    annotation[: -len(self.gene_set_file_suffix)] + ".csv",
+                    annotations_subpath,
+                    True,
+                )
+                for annotation in sorted_files
+                if self.is_gene_set(annotation)
+                and annotation[: -len(self.gene_set_file_suffix)]
+                not in [a.name for a in annotation_files]
+                and os.path.isfile(os.path.join(annotations_fullpath, annotation))
+            ]
+
+            return sorted(annotation_files + gene_sets_files, key=lambda x: x.name)
         else:
             return None

cellxgene_gateway/items/s3/s3item_source.py

@@ -116,6 +116,9 @@ class S3ItemSource(ItemSource):
         branches = None
         if len(subdir_keys) > 0:
             branches = [self.scan_directory(key) for key in subdir_keys]
+            branches = [
+                branch for branch in branches if branch.items or branch.branches
+            ]
 
         return ItemTree(directory_key, items, branches)
 

cellxgene_gateway/subprocess_backend.py

@@ -9,16 +9,11 @@
 
 import logging
 import subprocess
-
+from http import HTTPStatus
-from flask_api import status
 
 from cellxgene_gateway.cache_entry import CacheEntryStatus
 from cellxgene_gateway.dir_util import make_annotations
-from cellxgene_gateway.env import (
+from cellxgene_gateway.env import cellxgene_args, enable_annotations, enable_backed_mode
-    cellxgene_args,
-    enable_annotations,
-    enable_backed_mode,
-)
 from cellxgene_gateway.process_exception import ProcessException
 
 logger = logging.getLogger(__name__)
@@ -80,10 +75,10 @@ class SubprocessBackend:
                     or "Could not open file" in stderr
                 ):
                     message = "File was invalid."
-                    http_status = status.HTTP_400_BAD_REQUEST
+                    http_status = HTTPStatus.BAD_REQUEST
                 else:
                     message = "Cellxgene failed to launch dataset."
-                    http_status = status.HTTP_500_INTERNAL_SERVER_ERROR
+                    http_status = HTTPStatus.INTERNAL_SERVER_ERROR
 
                 cache_entry.status = CacheEntryStatus.error
                 cache_entry.set_error(message, stderr, http_status)

environment.yml

@@ -2,7 +2,7 @@ name: cellxgene-gateway
 channels:
 - conda-forge
 dependencies:
-- python=3.9
+- python=3.11
 - requests
 - flask
 - psutil
@@ -13,6 +13,5 @@ dependencies:
 - pip
 - pip:
   - pre_commit
-  - flask-api
   - werkzeug
   - cellxgene

examples/customized_docker_image/Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.9
+
+RUN pip install cellxgene-gateway 'MarkupSafe<2.1'
+
+COPY customize_ui.sh customize_ui.sh
+RUN CELLXGENE_GATEWAY_DIR=/usr/local/lib/python3.9/site-packages/cellxgene_gateway . ./customize_ui.sh
+
+ENV CELLXGENE_DATA=/cellxgene-data
+ENV CELLXGENE_LOCATION=/usr/local/bin/cellxgene
+EXPOSE 5005
+
+RUN mkdir /cellxgene-data
+
+CMD ["cellxgene-gateway"]

examples/customized_docker_image/README.md

@@ -0,0 +1,14 @@
+# Purpose
+
+This is a simple example of how to make a small script to customize the UI of cellxgene-gateway. The script that does the customization is `customize_ui.sh`, it simply makes the main header green using CSS but you could do anything you want there (including adding more script tags, etc).
+
+# Usage
+
+```
+docker build -t cellxgene_custom .
+CELLXGENE_DATA=`pwd`/../../../cellxgene_data
+docker run -p 5005:5005 --mount src=$CELLXGENE_DATA,target=/cellxgene-data,type=bind cellxgene_custom
+```
+
+If you now open http://localhost:5005 you should see a green cellxgene gateway header.
+

examples/customized_docker_image/customize_ui.sh

@@ -0,0 +1,3 @@
+# make the header bright green
+find "${CELLXGENE_GATEWAY_DIR}/templates" -name index.html -exec sed -i -e 's/<head>/<head>\
+> <style> header h3 {color: #0F0;} <\/style>/g' {} \;

requirements.txt

@@ -1,6 +1,5 @@
 cellxgene
 flask
-flask-api
 werkzeug
 psutil
 requests

tests/test_filecrawl.py

@@ -1,5 +1,6 @@
 import unittest
-from unittest.mock import MagicMock, patch
+from collections import defaultdict
+from unittest.mock import patch
 
 from cellxgene_gateway.filecrawl import (
     render_item,
@@ -13,26 +14,79 @@ from cellxgene_gateway.items.item import ItemTree, ItemType
 source = FileItemSource("/tmp")
 
 
+def make_entry(subpath="somepath", annotations=None):
+    return FileItem(
+        subpath=subpath,
+        name="entry",
+        ext=".h5ad",
+        type=ItemType.h5ad,
+        annotations=annotations,
+    )
+
+
 class TestRenderEntry(unittest.TestCase):
     def test_GIVEN_path_both_slash_THEN_view_has_single_slash(self):
-        entry = FileItem(subpath="/somepath/", name="entry", type=ItemType.h5ad)
+        entry = make_entry(subpath="/somepath/")
         rendered = render_item(entry, source)
-        self.assertIn("view/somepath/entry/'", rendered)
+        self.assertIn("view/somepath/entry.h5ad/'", rendered)
 
     def test_GIVEN_path_starts_slash_THEN_view_has_single_slash(self):
-        entry = FileItem(subpath="/somepath", name="entry", type=ItemType.h5ad)
+        entry = make_entry(subpath="/somepath")
         rendered = render_item(entry, source)
-        self.assertIn("view/somepath/entry/'", rendered)
+        self.assertIn("view/somepath/entry.h5ad/'", rendered)
 
     def test_GIVEN_path_ends_slash_THEN_view_has_single_slash(self):
-        entry = FileItem(subpath="somepath/", name="entry", type=ItemType.h5ad)
+        entry = make_entry(subpath="somepath/")
         rendered = render_item(entry, source)
-        self.assertIn("view/somepath/entry/'", rendered)
+        self.assertIn("view/somepath/entry.h5ad/'", rendered)
 
     def test_GIVEN_path_no_slash_THEN_view_has_single_slash(self):
-        entry = FileItem(subpath="somepath", name="entry", type=ItemType.h5ad)
+        entry = make_entry(subpath="somepath")
         rendered = render_item(entry, source)
-        self.assertIn("view/somepath/entry/'", rendered)
+        self.assertIn("view/somepath/entry.h5ad/'", rendered)
+
+
+class TestRenderAnnotation(unittest.TestCase):
+    @patch("cellxgene_gateway.filecrawl.enable_annotations", new=True)
+    def test_GIVEN_no_annotation_THEN_new_alone(self):
+        entry = make_entry(annotations=None)
+        rendered = render_item(entry, source)
+        self.assertIn(
+            "> | annotations: <a class='new' href='/source/Files:/tmp/view/somepath/entry_annotations'>new</a></li>",
+            rendered,
+        )
+
+    @patch("cellxgene_gateway.filecrawl.enable_annotations", new=True)
+    def test_GIVEN_annotation_THEN_new_before(self):
+        annotation = FileItem(
+            subpath="somepath/entry_annotations",
+            name="annot",
+            ext=".csv",
+            type=ItemType.annotation,
+        )
+        entry = make_entry(annotations=[annotation])
+        rendered = render_item(entry, source)
+        self.assertIn(
+            "> | annotations: <a class='new' href='/source/Files:/tmp/view/somepath/entry_annotations'>new</a>,"
+            " <a href='/source/Files:/tmp/view/somepath/entry_annotations/annot.csv/'>annot</a></li>",
+            rendered,
+        )
+
+    @patch("cellxgene_gateway.filecrawl.enable_annotations", new=True)
+    def test_GIVEN_annotation_THEN_escaped(self):
+        annotation = FileItem(
+            subpath="somepath/entry_annotations",
+            name="hot&cold",
+            ext=".csv",
+            type=ItemType.annotation,
+        )
+        entry = make_entry(annotations=[annotation])
+        rendered = render_item(entry, source)
+        self.assertIn(
+            "> | annotations: <a class='new' href='/source/Files:/tmp/view/somepath/entry_annotations'>new</a>,"
+            " <a href='/source/Files:/tmp/view/somepath/entry_annotations/hot&cold.csv/'>hot&amp;cold</a></li>",
+            rendered,
+        )
 
 
 class TestRenderItemSource(unittest.TestCase):
@@ -48,12 +102,47 @@ class TestRenderItemSource(unittest.TestCase):
 
 
 class TestRenderItemTree(unittest.TestCase):
+    def setUp(self):
+        from cellxgene_gateway.gateway import app
+
+        self.app = app
+        self.app_context = self.app.test_request_context()
+        self.app_context.push()
+
     @patch("cellxgene_gateway.items.file.fileitem_source.FileItemSource")
     def test_GIVEN_deep_nested_dirs_THEN_includes_dirs_in_output(self, item_source):
         item_source.name = "FakeSource"
-        item_tree = ItemTree("foo/bar/baz", [], [])
+        item_source.get_annotations_subpath = lambda _: "FakeAnnotations"
+        file_item = FileItem(
+            subpath="foo/bar/baz", name="file.h5ad", type=ItemType.h5ad
+        )
+        item_tree = ItemTree("foo/bar/baz", [file_item], [])
         rendered = render_item_tree(item_tree, item_source)
         self.assertEqual(
             rendered,
-            "<li><a href='/filecrawl/foo/bar/baz?source=FakeSource'>baz</a><ul></ul></li>",
+            "<li><a href='/filecrawl/foo/bar/baz?source=FakeSource'>baz</a><ul>"
+            "<li> <a href='/source/FakeSource/view/foo/bar/baz/file.h5ad/'>file.h5ad</a>"
+            " </li></ul></li>",
+        )
+
+    @patch(
+        "os.listdir",
+        side_effect=lambda parent: defaultdict(
+            list, {"tmp": ["foo"], "tmp/foo": ["bar"]}
+        )[parent],
+    )
+    @patch("os.path.exists", return_value=True)
+    def test_GIVEN_dirs_without_h5ad_THEN_excludes_dirs_in_output(
+        self, listdir, exists
+    ):
+        # Directories:
+        # - tmp
+        #   - foo
+        #     - bar (no h5ad files)
+        item_source = FileItemSource("tmp", name="local")
+        item_tree = item_source.list_items("foo")
+        rendered = render_item_tree(item_tree, item_source)
+        self.assertEqual(
+            rendered,
+            "<li><a href='/filecrawl/foo?source=local'>foo</a><ul></ul></li>",
         )