#59 add temporary workaround for jinja

Force reload of s3 file structure on every request

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,67 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '18 6 * * 6'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'javascript', 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/pr-checks.yaml

@@ -39,6 +39,7 @@ jobs:
           conda env create -f environment.yml
           eval "$(conda shell.bash hook)"
           conda activate cellxgene-gateway
+          pip install markupsafe==2.0.1 # temporary workaround for jinja2-2.11.3 calling soft_unicode in markupsafe
           python setup.py install
 
       - name: Run tests
@@ -51,7 +52,7 @@ jobs:
         run: |
           eval "$(conda shell.bash hook)"
           conda activate cellxgene-gateway
-          coverage report --fail-under 47
+          coverage report --fail-under 41
           coverage xml -i
 
       - name: "Upload coverage to Codecov"

.pre-commit-config.yaml

@@ -8,7 +8,6 @@ repos:
       types: [python]
       stages: [commit]
     - id: black
-      language_version: python3.6+
       name: black
       language: system
       entry: black

Changelog.md

@@ -1,6 +1,47 @@
+# 0.3.8
+
+* Fixed bug #57 affecting deeply nested subdirectory listing
+
+# 0.3.7
+
+* added back /metadata/ip_address endpoint
+
+# 0.3.6
+
+* pinned version of werkzeug
+
+# 0.3.5
+
+* Pinned flask version to match cellxgene 0.17.0
+
+# 0.3.4
+
+* Fixed bug #50 affecting subdirectory listing
+
+# 0.3.3
+
+* Fixed bug #48 affecting cache pruning
+
+# 0.3.2
+
+ * Fixed bug #45 affecting multi-level S3 folders
+ * Added extra_scripts to cache_status page
+
+# 0.3.1
+
+ * Added missing __init__.py
+
+# 0.3.0
+
+ * Added support for itemsource interface, allowing s3 hosting
+ * Removed support for http file uploads
+ * Only set wsgi.url_scheme when EXTERNAL_PROTOCOL is set (see issue #43) 
+ * Dropped flake8 due to conflicts with black
+ * Added code coverage metrics
+
 # 0.2.3
 
-* Added support for ProxyFix
+ * Added support for ProxyFix
 
 # 0.2.2
 

README.md

@@ -2,7 +2,7 @@
 
 Cellxgene Gateway allows you to use the Cellxgene Server provided by the Chan Zuckerberg Institute (https://github.com/chanzuckerberg/cellxgene) with multiple datasets. It displays an index of available h5ad (anndata) files. When a user clicks on a file name, it launches a Cellxgene Server instance that loads that particular data file and once it is available  proxies requests to that server.
 
-[![codecov](https://codecov.io/gh/Novartis/cellxgene-gateway/branch/master/graph/badge.svg?token=ndEFSzRKJn)](https://codecov.io/gh/Novartis/cellxgene-gateway)
+[![codecov](https://codecov.io/gh/Novartis/cellxgene-gateway/branch/master/graph/badge.svg?token=ndEFSzRKJn)](https://codecov.io/gh/Novartis/cellxgene-gateway) [![PyPI](https://img.shields.io/pypi/v/cellxgene-gateway)](https://pypi.org/project/cellxgene-gateway/) [![PyPI - Downloads](https://img.shields.io/pypi/dm/cellxgene-gateway)](https://pypistats.org/packages/cellxgene-gateway)
 
 # Running locally
 
@@ -76,6 +76,7 @@ Optional environment variables:
 * `GATEWAY_EXTRA_SCRIPTS` - JSON array of script paths, will be embedded into each page and forwarded with `--scripts` to cellxgene server
 * `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations. 
 * `GATEWAY_ENABLE_BACKED_MODE` - Set to `true` or to `1` to load AnnData in file-backed mode. This saves memory and speeds up launch time but may reduce overall performance.
+* `S3_ENABLE_LISTINGS_CACHE` - Set to `true` or to `1` to cache listings of S3 folders for performance. Can be overridden by setting `filecrawl.html?refresh=true` query parameter.
 
 If any of the following optional variables are set, [ProxyFix](https://werkzeug.palletsprojects.com/en/1.0.x/middleware/proxy_fix/) will be used.
 * `PROXY_FIX_FOR` - Number of upstream proxies setting X-Forwarded-For

SECURITY.md

@@ -0,0 +1,15 @@
+# Security Policy
+
+## Supported Versions
+
+Use this section to tell people about which versions of your project are
+currently being supported with security updates.
+
+| Version | Supported          |
+| ------- | ------------------ |
+| 0.3.2   | :white_check_mark: |
+| <= 0.3.1   | :x:                |
+
+## Reporting a Vulnerability
+
+Please file a bug report issue.

cellxgene_gateway/__init__.py

@@ -7,4 +7,4 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
-__version__ = "0.2.3"
+__version__ = "0.3.8"

cellxgene_gateway/dir_util.py

@@ -14,44 +14,6 @@ from flask_api import status
 from cellxgene_gateway import env
 from cellxgene_gateway.cellxgene_exception import CellxgeneException
 
-
-def is_subdir(full_path, parent_path):
-    subdir = os.path.realpath(full_path)
-    parent = os.path.realpath(parent_path)
-    return subdir.startswith(parent)
-
-
-def create_dir(parent_path, dir_name):
-    full_path = os.path.join(parent_path, dir_name)
-
-    if "/" in dir_name:
-        raise CellxgeneException(
-            "Please have no slashes in the intended directory.",
-            status.HTTP_400_BAD_REQUEST,
-        )
-    elif not os.path.exists(parent_path):
-        raise CellxgeneException(
-            "The selected User directory does not exist.",
-            status.HTTP_400_BAD_REQUEST,
-        )
-    elif os.path.exists(full_path):
-        raise CellxgeneException(
-            "The provided subdirectory already exists within Directory.",
-            status.HTTP_400_BAD_REQUEST,
-        )
-    elif not is_subdir(full_path, parent_path):
-        raise CellxgeneException(
-            "The directory must be a subdirectory of the parent path.",
-            status.HTTP_400_BAD_REQUEST,
-        )
-    elif not os.path.isdir(parent_path):
-        raise CellxgeneException(
-            "The parent is not a directory.", status.HTTP_400_BAD_REQUEST
-        )
-    else:
-        os.mkdir(full_path)
-
-
 annotations_suffix = "_annotations"
 h5ad_suffix = ".h5ad"
 

cellxgene_gateway/env.py

@@ -20,7 +20,7 @@ external_host = os.environ.get(
     os.environ.get("GATEWAY_HOST", f"localhost:{gateway_port}"),
 )
 external_protocol = os.environ.get(
-    "EXTERNAL_PROTOCOL", os.environ.get("GATEWAY_PROTOCOL", "http")
+    "EXTERNAL_PROTOCOL", os.environ.get("GATEWAY_PROTOCOL", None)
 )
 ip = os.environ.get("GATEWAY_IP")
 extra_scripts = os.environ.get("GATEWAY_EXTRA_SCRIPTS")

cellxgene_gateway/extra_scripts.py

@@ -8,6 +8,7 @@
 # the specific language governing permissions and limitations under the License.
 
 from json import loads
+from json.decoder import JSONDecodeError
 
 from cellxgene_gateway import env
 
@@ -17,4 +18,9 @@ def get_extra_scripts():
     # ['https://www.googletagmanager.com/gtag/js?id=UA-123456-2',
     #  f"{env.external_protocol}://{env.external_host}/static/js/google_ua.js"]
     # where google_ua.js is a script you add to the static/js folder prior to deployment.
-    return [] if env.extra_scripts is None else loads(env.extra_scripts)
+    try:
+        return [] if env.extra_scripts is None else loads(env.extra_scripts)
+    except JSONDecodeError as exc:
+        raise Exception(
+            f'Error parsing GATEWAY_EXTRA_SCRIPTS, expected JSON array e.g. ["https://example.com/path/to/script.js"]'
+        ) from exc

cellxgene_gateway/filecrawl.py

@@ -54,7 +54,7 @@ def render_item_tree(item_tree, item_source):
     if item_tree.descriptor:
         descriptor = item_tree.descriptor.lstrip("/")
         url = f"/filecrawl/{descriptor}?source={item_source.name}"
-        name = descriptor.rsplit("/")[1] if descriptor.find("/") >= 0 else descriptor
+        name = descriptor.rsplit("/", 1)[-1]
         return f"<li><a href='{url}'>{name}</a>{html}</li>"
     else:
         return html
@@ -62,5 +62,6 @@ def render_item_tree(item_tree, item_source):
 
 def render_item_source(item_source, filter=None):
     item_tree = item_source.list_items(filter)
-    heading = f"<h6><a href='/filecrawl.html?source={urllib.parse.quote_plus(item_source.name)}'>{item_source.name}</a></h6>"
+    filterpart = "" if filter is None else ":" + filter
+    heading = f"<h6><a href='/filecrawl.html?source={urllib.parse.quote_plus(item_source.name)}'>{item_source.name}</a>{filterpart}</h6>"
     return heading + render_item_tree(item_tree, item_source)

cellxgene_gateway/gateway.py

@@ -22,16 +22,13 @@ from flask import (
     send_from_directory,
     url_for,
 )
-from flask_api import status
 from werkzeug.middleware.proxy_fix import ProxyFix
-from werkzeug.utils import secure_filename
 
 from cellxgene_gateway import env, flask_util
 from cellxgene_gateway.backend_cache import BackendCache
 from cellxgene_gateway.cache_entry import CacheEntryStatus
 from cellxgene_gateway.cache_key import CacheKey
 from cellxgene_gateway.cellxgene_exception import CellxgeneException
-from cellxgene_gateway.dir_util import create_dir, is_subdir
 from cellxgene_gateway.extra_scripts import get_extra_scripts
 from cellxgene_gateway.filecrawl import render_item_source
 from cellxgene_gateway.process_exception import ProcessException
@@ -46,12 +43,21 @@ default_item_source = None
 
 def _force_https(app):
     def wrapper(environ, start_response):
-        environ["wsgi.url_scheme"] = env.external_protocol
+        if env.external_protocol is not None:
+            environ["wsgi.url_scheme"] = env.external_protocol
         return app(environ, start_response)
 
     return wrapper
 
 
+def set_no_cache(resp):
+    resp.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
+    resp.headers["Pragma"] = "no-cache"
+    resp.headers["Expires"] = "0"
+    resp.headers["Cache-Control"] = "public, max-age=0"
+    return resp
+
+
 app.wsgi_app = _force_https(app.wsgi_app)
 if (
     env.proxy_fix_for > 0
@@ -157,10 +163,7 @@ def filecrawl(path=None):
             path=path,
         )
     )
-    resp.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
-    resp.headers["Pragma"] = "no-cache"
-    resp.headers["Expires"] = "0"
-    resp.headers["Cache-Control"] = "public, max-age=0"
+    set_no_cache(resp)
     return resp
 
 
@@ -209,14 +212,21 @@ def do_view(path, source_name=None):
         match.status == CacheEntryStatus.loaded
         or match.status == CacheEntryStatus.loading
     ):
-        return match.serve_content(path)
+        if source.is_authorized(match.key.descriptor):
+            return match.serve_content(path)
+        else:
+            raise CellxgeneException("User not authorized to access this data", 403)
     elif match.status == CacheEntryStatus.error:
         raise ProcessException.from_cache_entry(match)
 
 
 @app.route("/cache_status", methods=["GET"])
 def do_GET_status():
-    return render_template("cache_status.html", entry_list=cache.entry_list)
+    return render_template(
+        "cache_status.html",
+        entry_list=cache.entry_list,
+        extra_scripts=get_extra_scripts(),
+    )
 
 
 @app.route("/cache_status.json", methods=["GET"])
@@ -263,6 +273,12 @@ def do_terminate(path):
     return redirect(url_for("do_GET_status"), code=302)
 
 
+@app.route("/metadata/ip_address", methods=["GET"])
+def ip_address():
+    resp = make_response(env.ip)
+    return set_no_cache(resp)
+
+
 def launch():
     env.validate()
     if not item_sources or not len(item_sources):

cellxgene_gateway/items/file/fileitem_source.py

@@ -51,7 +51,7 @@ class FileItemSource(ItemSource):
         return self.convert_h5ad_path_to_annotation(item.descriptor)
 
     def list_items(self, filter: str = None) -> ItemTree:
-        item_tree = self.scan_directory()
+        item_tree = self.scan_directory("" if filter is None else filter)
 
         """def get_items(dir):
             if dir.branches:
@@ -121,6 +121,9 @@ class FileItemSource(ItemSource):
         if self.is_h5ad_file(full_path):
             return self.shallowitem_from_descriptor(descriptor)
 
+    def is_authorized(self, descriptor):
+        return True
+
     def lookup(self, indescriptor: str) -> LookupResult:
         descriptor = indescriptor.strip("/")
         if descriptor.endswith(self.annotation_file_suffix):

cellxgene_gateway/items/item_source.py

@@ -40,6 +40,10 @@ class ItemSource(ABC):
     def update(self, item: Item) -> None:
         raise Exception('"update" unimplemented')
 
+    @abstractmethod
+    def is_authorized(self, descriptor: str) -> bool:
+        raise Exception('"is_authorized" unimplemented')
+
     @abstractmethod
     def lookup(self, descriptor: str) -> LookupResult:
         raise Exception('"lookup" unimplemented')

cellxgene_gateway/items/s3/s3item_source.py

@@ -7,9 +7,11 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
-from os.path import basename, dirname, join
+import os
+from os.path import basename, dirname
 from typing import List
 
+import flask
 import s3fs
 
 from cellxgene_gateway import dir_util
@@ -18,6 +20,10 @@ from cellxgene_gateway.items.item_source import ItemSource, LookupResult
 from cellxgene_gateway.items.s3.s3item import S3Item
 
 
+def truthy(val: str):
+    return val.lower() in ["true", "1"]
+
+
 class S3ItemSource(ItemSource):
     def __init__(
         self,
@@ -28,7 +34,10 @@ class S3ItemSource(ItemSource):
         annotation_file_suffix=".csv",
     ):
         self._name = name
-        self.s3 = s3fs.S3FileSystem()
+        enable_cache = os.environ.get("S3_ENABLE_LISTINGS_CACHE", "false").lower()
+        assert enable_cache in ["0", "1", "false", "true"]
+        self.use_listings_cache = truthy(enable_cache)
+        self.s3 = s3fs.S3FileSystem(use_listings_cache=self.use_listings_cache)
         if bucket.startswith("s3://"):
             raise Exception(
                 f"Bucket name should not include s3:// prefix, got {bucket}"
@@ -38,8 +47,11 @@ class S3ItemSource(ItemSource):
         self.annotation_dir_suffix = annotation_dir_suffix
         self.annotation_file_suffix = annotation_file_suffix
 
-    def url(self, path):
-        return "s3://" + join(self.bucket, path)
+    def url(self, key):
+        return "s3://" + self.bucket + "/" + key
+
+    def remove_bucket(self, filepath):
+        return filepath[len(self.bucket) :].lstrip("/")
 
     @property
     def name(self):
@@ -61,49 +73,51 @@ class S3ItemSource(ItemSource):
         return self.convert_h5ad_key_to_annotation(item.descriptor)
 
     def list_items(self, filter: str = None) -> ItemTree:
-        item_tree = self.scan_directory()
+        item_tree = self.scan_directory("" if filter is None else filter)
         return item_tree
 
-    def scan_directory(self, subpath="") -> dict:
-        url = self.url(subpath)
+    @property
+    def refresh(self):
+        return (
+            truthy(flask.request.args.get("refresh", default="false"))
+            or not self.use_listings_cache
+        )
+
+    def scan_directory(self, directory_key="") -> dict:
+        url = self.url(directory_key)
 
         if not self.s3.exists(url):
             raise Exception(f"S3 url '{url}' does not exist.")
 
         s3key_map = dict(
-            (filepath[len(self.bucket) :].lstrip("/"), "s3://" + filepath)
-            for filepath in sorted(self.s3.ls(url))
+            (self.remove_bucket(filepath), "s3://" + filepath)
+            for filepath in sorted(self.s3.ls(url, refresh=self.refresh))
         )
 
         def is_annotation_dir(dir_s3key):
             return (
                 dir_s3key.endswith(self.annotation_dir_suffix)
-                and self.convert_annotation_key_to_h5ad(dir_s3key) in h5ad_paths
+                and self.convert_annotation_key_to_h5ad(dir_s3key) in h5ad_keys
             )
 
-        h5ad_paths = [
+        h5ad_keys = [
             filepath
             for filepath, item_url in s3key_map.items()
             if self.is_h5ad_url(item_url)
         ]
 
-        subdirs = [
+        subdir_keys = [
             filepath
             for filepath, item_url in s3key_map.items()
             if self.s3.isdir(item_url) and not is_annotation_dir(filepath)
         ]
 
-        items = [
-            self.make_s3item_from_key(filename, join(subpath, filename))
-            for filename in h5ad_paths
-        ]
+        items = [self.make_s3item_from_key(basename(key), key) for key in h5ad_keys]
         branches = None
-        if len(subdirs) > 0:
-            branches = [
-                self.scan_directory(join(subpath, subdir)) for subdir in subdirs
-            ]
+        if len(subdir_keys) > 0:
+            branches = [self.scan_directory(key) for key in subdir_keys]
 
-        return ItemTree(subpath, items, branches)
+        return ItemTree(directory_key, items, branches)
 
     def create_annotation(self, item: S3Item, name: str) -> S3Item:
         annotation = self.make_s3item_from_key(
@@ -115,6 +129,9 @@ class S3ItemSource(ItemSource):
     def update(self, item: S3Item) -> None:
         pass
 
+    def is_authorized(self, descriptor):
+        return True
+
     def lookup_item(self, descriptor):
         full_path = self.url(descriptor)
         if self.is_h5ad_url(full_path):
@@ -163,11 +180,13 @@ class S3ItemSource(ItemSource):
         if self.s3.isdir(annotations_fullpath):
             return [
                 self.make_s3item_from_key(
-                    annotation, join(annotations_subpath, annotation), True
+                    basename(annotation), self.remove_bucket(annotation), True
+                )
+                for annotation in sorted(
+                    self.s3.ls(annotations_fullpath, refresh=self.refresh)
                 )
-                for annotation in sorted(self.s3.ls(annotations_fullpath))
                 if annotation.endswith(self.annotation_file_suffix)
-                and self.s3.isfile(join(annotations_fullpath, annotation))
+                and self.s3.isfile("s3://" + annotation)
             ]
         else:
             return None

cellxgene_gateway/path_util.py

@@ -1,41 +0,0 @@
-# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
-# under the Apache License, Version 2.0 (the "License"); you may not use
-# this file except in compliance with the License. You may obtain a copy
-# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
-# required by applicable law or agreed to in writing, software distributed
-# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
-# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
-# the specific language governing permissions and limitations under the License.
-
-import os
-
-from flask_api import status
-
-from cellxgene_gateway.cache_key import CacheKey
-from cellxgene_gateway.cellxgene_exception import CellxgeneException
-from cellxgene_gateway.dir_util import make_h5ad
-
-
-def validate_exists(file_path):
-    if not os.path.exists(file_path):
-        raise CellxgeneException(
-            "File does not exist: " + file_path, status.HTTP_400_BAD_REQUEST
-        )
-
-
-def validate_is_file(file_path):
-    validate_exists(file_path)
-    if not os.path.isfile(file_path):
-        raise CellxgeneException(
-            "Path is not file: " + file_path, status.HTTP_400_BAD_REQUEST
-        )
-    return
-
-
-def validate_is_dir(file_path):
-    validate_exists(file_path)
-    if not os.path.isdir(file_path):
-        raise CellxgeneException(
-            "Path is not dir: " + file_path, status.HTTP_400_BAD_REQUEST
-        )
-    return

cellxgene_gateway/prune_process_cache.py

@@ -39,9 +39,9 @@ class PruneProcessCache:
 
         for process in processes_to_delete:
             try:
-                logger.info(f"pruning process {process.pid} ({process.key.dataset})")
+                logger.info(f"pruning process {process.pid} ({process.key.descriptor})")
                 self.cache.prune(process)
             except Exception:
                 logger.exception(
-                    "failed to prune process {process.pid} ({process.dataset})"
+                    "failed to prune process {process.pid} ({process.key.descriptor})"
                 )

cellxgene_gateway/templates/cache_status.html

@@ -75,7 +75,9 @@
                     const el = $(this);
                     const ts = el.text();
                     const dt = new Date(parseInt(ts * 1000));
-                    el.html(`${dt.toISOString()}<br>(${ts})`);
+                    el.prepend(`${dt.toISOString()}<br>(`);
+                    el.append(')');
+
                });
           })
      </script>

environment.yml

@@ -4,11 +4,15 @@ channels:
 dependencies:
 - python=3.7
 - requests
-- flask
+- flask<2.0.0,>=1.0.2
 - psutil
 - black
+- twine
+- isort
 - coverage
 - pip
 - pip:
-  - flask-api
+  - pre_commit
+  - flask-api==2.0
+  - werkzeug==1.0.1
   - cellxgene>=0.15

requirements.txt

@@ -1,5 +1,6 @@
 cellxgene>=0.15
-flask
-flask_api
+flask<2.0.0,>=1.0.2
+flask-api==2.0
+werkzeug==1.0.1
 psutil
 requests

tests/items/file/test_fileitem_source.py

@@ -1,10 +1,31 @@
 import tempfile
 import unittest
+from unittest.mock import patch
 
 from cellxgene_gateway.items.file.fileitem_source import FileItemSource
 
 
+def stub_join(path):
+    path.join = lambda x, y: x + "/" + y
+
+
 class TestFileItemSource(unittest.TestCase):
+    @patch("os.path")
+    @patch("os.listdir")
+    def test_list_items_GIVEN_no_subpath_THEN_checks_dir(self, listdir, path):
+        stub_join(path)
+        source = FileItemSource("/tmp/unittest", "local")
+        source.list_items()
+        path.exists.assert_called_once_with("/tmp/unittest/")
+
+    @patch("os.path")
+    @patch("os.listdir")
+    def test_list_items_GIVEN_subpath_THEN_checks_subpath(self, listdir, path):
+        stub_join(path)
+        source = FileItemSource("/tmp/unittest", "local")
+        source.list_items("foo")
+        path.exists.assert_called_once_with("/tmp/unittest/foo")
+
     def test_make_fileitem_from_path_GIVEN_annotation_file_THEN_name_lacks_csv(
         self,
     ):

tests/items/s3/test_s3item_source.py

@@ -0,0 +1,174 @@
+import unittest
+from unittest.mock import MagicMock, Mock, patch
+
+from cellxgene_gateway.items.item import ItemType
+from cellxgene_gateway.items.s3.s3item import S3Item
+from cellxgene_gateway.items.s3.s3item_source import S3ItemSource
+
+
+class TestScanDirectory(unittest.TestCase):
+    @patch("s3fs.S3FileSystem")
+    def test_GIVEN_invalid_bucket_THEN_throws_error(self, s3func):
+        class S3Mock:
+            def exists(path):
+                if path in ["s3://my-bucket/"]:
+                    return False
+
+        s3func.return_value = S3Mock
+        source = S3ItemSource("my-bucket")
+        with self.assertRaises(Exception) as context:
+            source.scan_directory()
+        self.assertEqual(
+            "S3 url 's3://my-bucket/' does not exist.",
+            str(context.exception),
+        )
+
+    @patch("s3fs.S3FileSystem")
+    @patch("flask.request")
+    def test__GIVEN_multilevel_bucket_THEN_properly_recurses_suburls(
+        self, requestMock, s3func
+    ):
+        class S3Mock:
+            def exists(path):
+                if path in [
+                    "s3://my-bucket/",
+                    "s3://my-bucket/pbmc3k.h5ad",
+                    "s3://my-bucket/lvl1",
+                    "s3://my-bucket/lvl1/pbmc3k_l1.h5ad",
+                    "s3://my-bucket/lvl1/lvl2",
+                    "s3://my-bucket/lvl1/lvl2/pbmc3k_l2.h5ad",
+                ]:
+                    return True
+                raise Exception("exists called with " + path)
+
+            def ls(path, refresh):
+                assert refresh == True
+                if path == "s3://my-bucket/":
+                    return [
+                        "my-bucket/lvl1",
+                        "my-bucket/pbmc3k.h5ad",
+                        "my-bucket/pbmc3k_annotations",
+                    ]
+                elif path == "s3://my-bucket/pbmc3k_annotations":
+                    return ["my-bucket/pbmc3k_annotations/annot.csv"]
+                elif path == "s3://my-bucket/lvl1":
+                    return ["my-bucket/lvl1/lvl2", "my-bucket/lvl1/pbmc3k_l1.h5ad"]
+                elif path == "s3://my-bucket/lvl1/lvl2":
+                    return ["my-bucket/lvl1/lvl2/pbmc3k_l2.h5ad"]
+
+                raise Exception("ls called with " + path)
+
+            def isdir(path):
+                if path in [
+                    "s3://my-bucket/lvl1",
+                    "s3://my-bucket/pbmc3k_annotations",
+                    "s3://my-bucket/lvl1/lvl2",
+                ]:
+                    return True
+                if path in [
+                    "s3://my-bucket/pbmc3k.h5ad",
+                    "s3://my-bucket/lvl1/pbmc3k_l1.h5ad",
+                    "s3://my-bucket/lvl1/pbmc3k_l1_annotations",
+                    "s3://my-bucket/lvl1/lvl2/pbmc3k_l2.h5ad",
+                    "s3://my-bucket/lvl1/lvl2/pbmc3k_l2_annotations",
+                ]:
+                    return False
+                raise Exception("isdir called with " + path)
+
+            def isfile(path):
+                if path in ["s3://my-bucket/pbmc3k_annotations/annot.csv"]:
+                    return True
+                if path in ["s3://my-bucket/pbmc3k_annotations"]:
+                    return False
+                raise Exception("isfile called with " + path)
+
+        s3func.return_value = S3Mock
+        requestMock.args.get.return_value = "true"
+        source = S3ItemSource("my-bucket")
+        tree = source.scan_directory()
+
+        def s3item_compare(i1, i2, msg=""):
+            self.assertEqual(i1.name, i2.name, "name equals")
+            self.assertEqual(i1.type, i2.type, "type equals")
+            self.assertEqual(i1.s3key, i2.s3key, "s3key equals")
+            if i1.annotations is None:
+                self.assertEqual(i1.annotations, i2.annotations, "annotations equals")
+            else:
+                self.assertEqual(
+                    len(i1.annotations),
+                    len(i2.annotations),
+                    "annotations length equals",
+                )
+                for a1, a2 in zip(i1.annotations, i2.annotations):
+                    self.assertEqual(a1, a2)
+            return True
+
+        self.addTypeEqualityFunc(S3Item, s3item_compare)
+
+        def assertTree(t, descriptor, items):
+            self.assertEqual(t.descriptor, descriptor)
+            self.assertEqual(len(t.items), len(items))
+            for i1, i2 in zip(t.items, items):
+                self.assertEqual(i1, i2)
+
+        assertTree(
+            tree,
+            "",
+            [
+                S3Item(
+                    "pbmc3k.h5ad",
+                    name="pbmc3k.h5ad",
+                    type=ItemType.h5ad,
+                    annotations=[
+                        S3Item(
+                            "pbmc3k_annotations/annot.csv",
+                            name="annot.csv",
+                            type=ItemType.annotation,
+                        )
+                    ],
+                )
+            ],
+        )
+        self.assertEqual(len(tree.branches), 1)
+        lvl1 = tree.branches[0]
+        assertTree(
+            lvl1,
+            "lvl1",
+            [
+                S3Item(
+                    "lvl1/pbmc3k_l1.h5ad",
+                    name="pbmc3k_l1.h5ad",
+                    type=ItemType.h5ad,
+                    annotations=None,
+                )
+            ],
+        )
+        self.assertEqual(len(lvl1.branches), 1)
+        lvl2 = lvl1.branches[0]
+        assertTree(
+            lvl2,
+            "lvl1/lvl2",
+            [
+                S3Item(
+                    "lvl1/lvl2/pbmc3k_l2.h5ad",
+                    name="pbmc3k_l2.h5ad",
+                    type=ItemType.h5ad,
+                    annotations=None,
+                )
+            ],
+        )
+        self.assertEqual(lvl2.branches, None)
+
+
+class TestListItems(unittest.TestCase):
+    def test_GIVEN_filter_THEN_pass_filter_into_scan_directory(self):
+        source = S3ItemSource("my-bucket")
+        source.scan_directory = MagicMock()
+        tree = source.list_items("some-filter")
+        source.scan_directory.assert_called_once_with("some-filter")
+
+    def test_GIVEN_no_filter_THEN_pass_empty_string_into_scan_directory(self):
+        source = S3ItemSource("my-bucket")
+        source.scan_directory = MagicMock()
+        tree = source.list_items()
+        source.scan_directory.assert_called_once_with("")

tests/test_backend_cache.py

@@ -0,0 +1,28 @@
+import unittest
+from unittest.mock import MagicMock, patch
+
+from cellxgene_gateway.backend_cache import is_port_in_use
+
+
+class TestIsPortInUse(unittest.TestCase):
+    @patch("socket.socket")
+    def test_GIVEN_free_port_THEN_returns_true(self, socketMock):
+        connectMock = socketMock()
+        connectMock.connect_ex.return_value = 0
+        connectMock.__enter__.return_value = connectMock
+        self.assertEqual(is_port_in_use(123), True)
+        self.assertTrue(connectMock.__enter__.calledOnce)
+        self.assertTrue(connectMock.__exit__.calledOnce)
+        self.assertTrue(connectMock.connect_ex.calledOnceWith("a"))
+        self.assertTrue(socketMock.calledOnceWith("a"))
+
+    @patch("socket.socket")
+    def test_GIVEN_used_port_THEN_returns_false(self, socketMock):
+        connectMock = socketMock()
+        connectMock.__enter__.return_value = connectMock
+        connectMock.connect_ex.return_value = 1
+        self.assertTrue(connectMock.__enter__.calledOnce)
+        self.assertTrue(connectMock.__exit__.calledOnce)
+        self.assertTrue(connectMock.connect_ex.calledOnceWith("a"))
+        self.assertTrue(socketMock.calledOnceWith("a"))
+        self.assertEqual(is_port_in_use(123), False)

tests/test_dir_util.py

@@ -0,0 +1,35 @@
+import unittest
+from unittest.mock import MagicMock, patch
+
+from cellxgene_gateway.dir_util import ensure_dir_exists, make_annotations, make_h5ad
+
+
+class TestMakeH5ad(unittest.TestCase):
+    def test_GIVEN_annotation_dir_THEN_returns_h5ad(self):
+        self.assertEqual(make_h5ad("pbmc_annotations"), "pbmc.h5ad")
+
+
+class TestMakeAnnotations(unittest.TestCase):
+    def test_GIVEN_h5ad_THEN_returns_annotations(self):
+        self.assertEqual(make_annotations("pbmc.h5ad"), "pbmc_annotations")
+
+
+class TestMakeAnnotations(unittest.TestCase):
+    def test_GIVEN_h5ad_THEN_returns_annotations(self):
+        self.assertEqual(make_annotations("pbmc.h5ad"), "pbmc_annotations")
+
+
+class TestEnsureDirExists(unittest.TestCase):
+    @patch("os.path.exists")
+    @patch("os.makedirs")
+    def test_GIVEN_existing_THEN_does_not_call_makedir(self, makedirsMock, existsMock):
+        existsMock.return_value = True
+        ensure_dir_exists("/foo")
+        makedirsMock.assert_not_called()
+
+    @patch("os.path.exists")
+    @patch("os.makedirs")
+    def test_GIVEN_not_existing_THEN_calls_makedir(self, makedirsMock, existsMock):
+        existsMock.return_value = False
+        ensure_dir_exists("/foo")
+        makedirsMock.assert_called_once_with("/foo")

tests/test_extra_scripts.py

@@ -21,6 +21,15 @@ class TestExtraScripts(unittest.TestCase):
     def test_GIVEN_empty_string_THEN_returns_empty_array(self):
         self.assertEqual(get_extra_scripts(), [])
 
+    @patch("cellxgene_gateway.env.extra_scripts", new="'asdf'")
+    def test_GIVEN_bare_string_THEN_throws_Exception(self):
+        with self.assertRaises(Exception) as context:
+            self.assertEqual(get_extra_scripts(), [])
+        self.assertEqual(
+            'Error parsing GATEWAY_EXTRA_SCRIPTS, expected JSON array e.g. ["https://example.com/path/to/script.js"]',
+            str(context.exception),
+        )
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_filecrawl.py

@@ -1,10 +1,14 @@
 import unittest
 from unittest.mock import MagicMock, patch
 
-from cellxgene_gateway.filecrawl import render_item
+from cellxgene_gateway.filecrawl import (
+    render_item,
+    render_item_source,
+    render_item_tree,
+)
 from cellxgene_gateway.items.file.fileitem import FileItem
 from cellxgene_gateway.items.file.fileitem_source import FileItemSource
-from cellxgene_gateway.items.item import ItemType
+from cellxgene_gateway.items.item import ItemTree, ItemType
 
 source = FileItemSource("/tmp")
 
@@ -29,3 +33,27 @@ class TestRenderEntry(unittest.TestCase):
         entry = FileItem(subpath="somepath", name="entry", type=ItemType.h5ad)
         rendered = render_item(entry, source)
         self.assertIn("view/somepath/entry/'", rendered)
+
+
+class TestRenderItemSource(unittest.TestCase):
+    @patch("cellxgene_gateway.items.file.fileitem_source.FileItemSource")
+    def test_GIVEN_some_filter_THEN_includes_filterpart_in_heading(self, item_source):
+        item_source.name = "FakeSource"
+        item_source.list_items.return_value = ItemTree("rootdir", [], [])
+        rendered = render_item_source(item_source, "some_filter")
+        self.assertEqual(
+            rendered,
+            "<h6><a href='/filecrawl.html?source=FakeSource'>FakeSource</a>:some_filter</h6><li><a href='/filecrawl/rootdir?source=FakeSource'>rootdir</a><ul></ul></li>",
+        )
+
+
+class TestRenderItemTree(unittest.TestCase):
+    @patch("cellxgene_gateway.items.file.fileitem_source.FileItemSource")
+    def test_GIVEN_deep_nested_dirs_THEN_includes_dirs_in_output(self, item_source):
+        item_source.name = "FakeSource"
+        item_tree = ItemTree("foo/bar/baz", [], [])
+        rendered = render_item_tree(item_tree, item_source)
+        self.assertEqual(
+            rendered,
+            "<li><a href='/filecrawl/foo/bar/baz?source=FakeSource'>baz</a><ul></ul></li>",
+        )

tests/test_prune_process_cache.py

@@ -1,8 +1,16 @@
 import unittest
-from unittest.mock import MagicMock, patch
+from unittest.mock import patch, seal
 
 from cellxgene_gateway.backend_cache import BackendCache
-from cellxgene_gateway.cache_entry import CacheEntry
+from cellxgene_gateway.cache_key import CacheKey
+from cellxgene_gateway.items.file.fileitem import FileItem
+from cellxgene_gateway.items.file.fileitem_source import FileItemSource
+from cellxgene_gateway.items.item import ItemType
+
+key = CacheKey(
+    FileItem("/czi/", name="pbmc3k.h5ad", type=ItemType.h5ad),
+    FileItemSource("/tmp", "local"),
+)
 
 
 class TestPruneProcessCache(unittest.TestCase):
@@ -15,14 +23,23 @@ class TestPruneProcessCache(unittest.TestCase):
 
         cache = BackendCache()
         old.timestamp = -100
+        old.foo = 12
+        old.pid = 1
+        old.key = key
+        old.terminate.return_value = None
+        seal(old)
+        new.key = key
         cache.entry_list.append(old)
         new.timestamp = -5
+        seal(new)
         cache.entry_list.append(new)
         self.assertEqual(len(cache.entry_list), 2)
         ppc = PruneProcessCache(cache)
         ppc.prune()
         self.assertEqual(len(cache.entry_list), 1)
         self.assertEqual(cache.entry_list[0], new)
+        self.assertEqual(cache.entry_list[0], new)
+        self.assertTrue(old.terminate.called)
 
 
 if __name__ == "__main__":