#50 fixed bug in listing subdirs

Fix cache pruning

.coveragerc

@@ -0,0 +1,13 @@
+[run]
+branch = True
+source = cellxgene_gateway
+
+[report]
+exclude_lines =
+    if self.debug:
+    pragma: no cover
+    raise NotImplementedError
+    if __name__ == .__main__.:
+ignore_errors = True
+omit =
+    tests/*

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,67 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '18 6 * * 6'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'javascript', 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/pr-checks.yaml

@@ -0,0 +1,67 @@
+# Tests that run on every PR
+
+name: Pull Request Checks
+
+on: [push, pull_request]
+
+jobs:
+  black:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+        name: Checkout repository
+
+      - uses: actions/setup-python@v2
+        name: Setup Python
+        with:
+          python-version: 3.9
+
+      - name: Install black
+        run: |
+          python -m pip install --upgrade pip
+          pip install black
+      - name: Run black
+        run: |
+          black . --check
+  # This job is copied over from `deploy.yaml`
+  run-tests:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+
+      # See: https://github.com/marketplace/actions/setup-conda
+      - uses: s-weigand/setup-conda@v1
+        with:
+          conda-channels: "conda-forge"
+
+      - name: Build environment
+        run: |
+          conda env create -f environment.yml
+          eval "$(conda shell.bash hook)"
+          conda activate cellxgene-gateway
+          python setup.py install
+
+      - name: Run tests
+        run: |
+          eval "$(conda shell.bash hook)"
+          conda activate cellxgene-gateway
+          coverage run -m unittest discover tests
+
+      - name: Check coverage
+        run: |
+          eval "$(conda shell.bash hook)"
+          conda activate cellxgene-gateway
+          coverage report --fail-under 41
+          coverage xml -i
+
+      - name: "Upload coverage to Codecov"
+        uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./coverage.xml
+          flags: unittests
+          env_vars: OS,PYTHON
+          name: codecov-umbrella
+          fail_ci_if_error: true
+          path_to_write_report: ./codecov_report.txt
+          verbose: true

.gitignore

@@ -55,6 +55,7 @@ htmlcov/
 .nox/
 .coverage
 .coverage.*
+htmlcov
 .cache
 nosetests.xml
 coverage.xml

.pre-commit-config.yaml

@@ -7,12 +7,6 @@ repos:
       language: system
       types: [python]
       stages: [commit]
-    - id: flake8
-      name: flake8
-      language: system
-      entry: flake8
-      types: [python]
-      stages: [commit]
     - id: black
       language_version: python3.6+
       name: black

Changelog.md

@@ -1,3 +1,28 @@
+# 0.3.3
+
+* Fixed bug #48 affecting cache pruning
+
+# 0.3.2
+
+ * Fixed bug #45 affecting multi-level S3 folders
+ * Added extra_scripts to cache_status page
+
+# 0.3.1
+
+ * Added missing __init__.py
+
+# 0.3.0
+
+ * Added support for itemsource interface, allowing s3 hosting
+ * Removed support for http file uploads
+ * Only set wsgi.url_scheme when EXTERNAL_PROTOCOL is set (see issue #43) 
+ * Dropped flake8 due to conflicts with black
+ * Added code coverage metrics
+
+# 0.2.3
+
+ * Added support for ProxyFix
+
 # 0.2.2
 
   * Fixed bug with annotations (missing annotation.js asset)

README.md

@@ -2,6 +2,8 @@
 
 Cellxgene Gateway allows you to use the Cellxgene Server provided by the Chan Zuckerberg Institute (https://github.com/chanzuckerberg/cellxgene) with multiple datasets. It displays an index of available h5ad (anndata) files. When a user clicks on a file name, it launches a Cellxgene Server instance that loads that particular data file and once it is available  proxies requests to that server.
 
+[![codecov](https://codecov.io/gh/Novartis/cellxgene-gateway/branch/master/graph/badge.svg?token=ndEFSzRKJn)](https://codecov.io/gh/Novartis/cellxgene-gateway) [![PyPI](https://img.shields.io/pypi/v/cellxgene-gateway)](https://pypi.org/project/cellxgene-gateway/) [![PyPI - Downloads](https://img.shields.io/pypi/dm/cellxgene-gateway)](https://pypistats.org/packages/cellxgene-gateway)
+
 # Running locally
 
 ## Prequisites
@@ -75,6 +77,13 @@ Optional environment variables:
 * `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations. 
 * `GATEWAY_ENABLE_BACKED_MODE` - Set to `true` or to `1` to load AnnData in file-backed mode. This saves memory and speeds up launch time but may reduce overall performance.
 
+If any of the following optional variables are set, [ProxyFix](https://werkzeug.palletsprojects.com/en/1.0.x/middleware/proxy_fix/) will be used.
+* `PROXY_FIX_FOR` - Number of upstream proxies setting X-Forwarded-For
+* `PROXY_FIX_PROTO` - Number of upstream proxies setting X-Forwarded-Proto
+* `PROXY_FIX_HOST` - Number of upstream proxies setting X-Forwarded-Host
+* `PROXY_FIX_PORT` - Number of upstream proxies setting X-Forwarded-Port
+* `PROXY_FIX_PREFIX` - Number of upstream proxies setting X-Forwarded-Prefix
+
 The defaults should be fine if you set up a venv and cellxgene_data folder as above.
 
 # Customization
@@ -117,6 +126,14 @@ python setup.py develop
 
 For convenience, the code repo includes a `run.sh.example` shell script to run the gateway.
 
+4. Install pre-commit hooks
+
+```bash
+conda install -c conda-forge pre-commit
+pre-commit install
+```
+
+
 ## Running Tests
 
 [![Build Status](https://travis-ci.org/Novartis/cellxgene-gateway.svg?branch=master)](https://travis-ci.org/Novartis/cellxgene-gateway)
@@ -125,6 +142,12 @@ For convenience, the code repo includes a `run.sh.example` shell script to run t
     python -m unittest discover tests
 ```
 
+## Code Coverage
+```bash
+    coverage run -m unittest discover tests
+    coverage html
+```
+
 ## Running Linters
 
 pip install isort flake8 black

SECURITY.md

@@ -0,0 +1,15 @@
+# Security Policy
+
+## Supported Versions
+
+Use this section to tell people about which versions of your project are
+currently being supported with security updates.
+
+| Version | Supported          |
+| ------- | ------------------ |
+| 0.3.2   | :white_check_mark: |
+| <= 0.3.1   | :x:                |
+
+## Reporting a Vulnerability
+
+Please file a bug report issue.

cellxgene_gateway/__init__.py

@@ -7,4 +7,4 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
-__version__ = "0.2.2"
+__version__ = "0.3.3"

cellxgene_gateway/backend_cache.py

@@ -9,6 +9,7 @@
 
 import time
 from threading import Thread
+from typing import List
 
 from flask_api import status
 
@@ -17,7 +18,6 @@ from cellxgene_gateway.cache_entry import CacheEntry, CacheEntryStatus
 from cellxgene_gateway.cache_key import CacheKey
 from cellxgene_gateway.cellxgene_exception import CellxgeneException
 from cellxgene_gateway.subprocess_backend import SubprocessBackend
-from typing import List
 
 process_backend = SubprocessBackend()
 

cellxgene_gateway/cache_entry.py

@@ -8,13 +8,14 @@
 # the specific language governing permissions and limitations under the License.
 import datetime
 import logging
+import re
 import urllib.parse
 from enum import Enum
 
 import psutil
 from flask import make_response, render_template, request
+from flask.wrappers import Response
 from requests import get, post, put
-import re
 
 from cellxgene_gateway import env
 from cellxgene_gateway.cellxgene_exception import CellxgeneException
@@ -110,9 +111,7 @@ class CacheEntry:
             except psutil.NoSuchProcess:
                 pass
 
-            logging.getLogger("cellxgene_gateway").info(
+            logging.getLogger("cellxgene_gateway").info(f"terminated {terminated}")
-                f"terminated {terminated}"
-            )
         self.status = CacheEntryStatus.terminated
 
     def rewrite_text_content(self, cellxgene_content):
@@ -120,30 +119,22 @@ class CacheEntry:
         gateway_content = (
             re.sub(
                 '(="|\()/static/',
-                f"\\1{self.gateway_basepath()}static/",
+                f"\\1{self.key.gateway_basepath()}static/",
                 cellxgene_content,
             )
             .replace("http://fonts.gstatic.com", "https://fonts.gstatic.com")
-            .replace(self.cellxgene_basepath(), self.gateway_basepath())
+            .replace(self.cellxgene_basepath(), self.key.gateway_basepath())
         )
         return gateway_content
 
-    def gateway_basepath(self):
-        source_path = (
-            f"/source/{urllib.parse.quote_plus(self.source_name)}"
-            if self.source_name
-            else ""
-        )
-        return f"{env.external_protocol}://{env.external_host}{source_path}/view/{self.key.descriptor}/"
-
     def cellxgene_basepath(self):
         return f"http://127.0.0.1:{self.port}"
 
     def serve_content(self, path):
-        gateway_basepath = self.gateway_basepath()
+        gateway_basepath = self.key.gateway_basepath()
         subpath = path[len(self.key.descriptor) :]  # noqa: E203
         if len(subpath) == 0:
-            r = make_response(f"Redirect to {gateway_basepath}\n", 301)
+            r = make_response(f"Redirect to {gateway_basepath}\n", 302)
             r.headers["location"] = gateway_basepath + querystring()
             return r
         elif self.status == CacheEntryStatus.loading:
@@ -193,9 +184,7 @@ class CacheEntry:
                 data=request.data,
             )
         else:
-            raise CellxgeneException(
+            raise CellxgeneException(f"Unexpected method {request.method}", 400)
-                f"Unexpected method {request.method}", 400
-            )
         content_type = cellxgene_response.headers["content-type"]
         if "text" in content_type:
             gateway_content = self.rewrite_text_content(

cellxgene_gateway/cache_key.py

@@ -15,6 +15,7 @@
 # 3) somedir/dataset_annotations: an annotation directory. The corresponding h5ad must exist, but the directory may not.
 #    in this case, descriptor == 'somedir/dataset_annotations', dataset == 'somedir/dataset.h5ad'
 
+from cellxgene_gateway import flask_util
 from cellxgene_gateway.items.item import Item
 from cellxgene_gateway.items.item_source import ItemSource, LookupResult
 
@@ -38,6 +39,16 @@ class CacheKey:
         else:
             return self.source.get_local_path(self.annotation_item)
 
+    def relaunch_url(self):
+        return flask_util.relaunch_url(self.descriptor, self.source_name)
+
+    def gateway_basepath(self):
+        return self.view_url + "/"
+
+    @property
+    def view_url(self):
+        return flask_util.view_url(self.descriptor, self.source_name)
+
     @property
     def source_name(self):
         return self.source.name

cellxgene_gateway/dir_util.py

@@ -14,44 +14,6 @@ from flask_api import status
 from cellxgene_gateway import env
 from cellxgene_gateway.cellxgene_exception import CellxgeneException
 
-
-def is_subdir(full_path, parent_path):
-    subdir = os.path.realpath(full_path)
-    parent = os.path.realpath(parent_path)
-    return subdir.startswith(parent)
-
-
-def create_dir(parent_path, dir_name):
-    full_path = os.path.join(parent_path, dir_name)
-
-    if "/" in dir_name:
-        raise CellxgeneException(
-            "Please have no slashes in the intended directory.",
-            status.HTTP_400_BAD_REQUEST,
-        )
-    elif not os.path.exists(parent_path):
-        raise CellxgeneException(
-            "The selected User directory does not exist.",
-            status.HTTP_400_BAD_REQUEST,
-        )
-    elif os.path.exists(full_path):
-        raise CellxgeneException(
-            "The provided subdirectory already exists within Directory.",
-            status.HTTP_400_BAD_REQUEST,
-        )
-    elif not is_subdir(full_path, parent_path):
-        raise CellxgeneException(
-            "The directory must be a subdirectory of the parent path.",
-            status.HTTP_400_BAD_REQUEST,
-        )
-    elif not os.path.isdir(parent_path):
-        raise CellxgeneException(
-            "The parent is not a directory.", status.HTTP_400_BAD_REQUEST
-        )
-    else:
-        os.mkdir(full_path)
-
-
 annotations_suffix = "_annotations"
 h5ad_suffix = ".h5ad"
 
@@ -62,3 +24,8 @@ def make_h5ad(el):
 
 def make_annotations(el):
     return el[:-5] + annotations_suffix
+
+
+def ensure_dir_exists(file_path):
+    if not os.path.exists(file_path):
+        os.makedirs(file_path)

cellxgene_gateway/env.py

@@ -12,7 +12,7 @@ import os
 import socket
 
 cellxgene_location = os.environ.get("CELLXGENE_LOCATION")
-cellxgene_data = os.environ.get("CELLXGENE_DATA")
+cellxgene_data = os.environ.get("CELLXGENE_DATA", "")
 cellxgene_args = os.environ.get("CELLXGENE_ARGS", None)
 gateway_port = int(os.environ.get("GATEWAY_PORT", "5005"))
 external_host = os.environ.get(
@@ -20,20 +20,16 @@ external_host = os.environ.get(
     os.environ.get("GATEWAY_HOST", f"localhost:{gateway_port}"),
 )
 external_protocol = os.environ.get(
-    "EXTERNAL_PROTOCOL", os.environ.get("GATEWAY_PROTOCOL", "http")
+    "EXTERNAL_PROTOCOL", os.environ.get("GATEWAY_PROTOCOL", None)
 )
 ip = os.environ.get("GATEWAY_IP")
 extra_scripts = os.environ.get("GATEWAY_EXTRA_SCRIPTS")
 ttl = os.environ.get("GATEWAY_TTL")
-enable_annotations = os.environ.get(
+enable_annotations = os.environ.get("GATEWAY_ENABLE_ANNOTATIONS", "").lower() in [
-    "GATEWAY_ENABLE_ANNOTATIONS", ""
-).lower() in [
     "true",
     "1",
 ]
-enable_backed_mode = os.environ.get(
+enable_backed_mode = os.environ.get("GATEWAY_ENABLE_BACKED_MODE", "").lower() in [
-    "GATEWAY_ENABLE_BACKED_MODE", ""
-).lower() in [
     "true",
     "1",
 ]
@@ -42,6 +38,12 @@ env_vars = {
     "CELLXGENE_LOCATION": cellxgene_location,
 }
 
+proxy_fix_for = int(os.environ.get("PROXY_FIX_FOR", "0"))
+proxy_fix_proto = int(os.environ.get("PROXY_FIX_PROTO", "0"))
+proxy_fix_host = int(os.environ.get("PROXY_FIX_HOST", "0"))
+proxy_fix_port = int(os.environ.get("PROXY_FIX_PORT", "0"))
+proxy_fix_prefix = int(os.environ.get("PROXY_FIX_PREFIX", "0"))
+
 optional_env_vars = {
     "EXTERNAL_HOST": external_host,
     "EXTERNAL_PROTOCOL": external_protocol,
@@ -53,6 +55,11 @@ optional_env_vars = {
     "GATEWAY_ENABLE_BACKED_MODE": enable_backed_mode,
     "CELLXGENE_ARGS": cellxgene_args,
     "CELLXGENE_DATA": cellxgene_data,
+    "PROXY_FIX_FOR": proxy_fix_for,
+    "PROXY_FIX_PROTO": proxy_fix_proto,
+    "PROXY_FIX_HOST": proxy_fix_host,
+    "PROXY_FIX_PORT": proxy_fix_port,
+    "PROXY_FIX_PREFIX": proxy_fix_prefix,
 }
 
 

cellxgene_gateway/extra_scripts.py

@@ -8,6 +8,7 @@
 # the specific language governing permissions and limitations under the License.
 
 from json import loads
+from json.decoder import JSONDecodeError
 
 from cellxgene_gateway import env
 
@@ -17,4 +18,9 @@ def get_extra_scripts():
     # ['https://www.googletagmanager.com/gtag/js?id=UA-123456-2',
     #  f"{env.external_protocol}://{env.external_host}/static/js/google_ua.js"]
     # where google_ua.js is a script you add to the static/js folder prior to deployment.
+    try:
         return [] if env.extra_scripts is None else loads(env.extra_scripts)
+    except JSONDecodeError as exc:
+        raise Exception(
+            f'Error parsing GATEWAY_EXTRA_SCRIPTS, expected JSON array e.g. ["https://example.com/path/to/script.js"]'
+        ) from exc

cellxgene_gateway/filecrawl.py

@@ -10,14 +10,20 @@
 import os
 import urllib.parse
 
+from cellxgene_gateway import env, flask_util
+from cellxgene_gateway.cache_key import CacheKey
+from cellxgene_gateway.dir_util import annotations_suffix, make_annotations, make_h5ad
+
 
 def render_annotations(item, item_source):
-    subpath = f"/source/{urllib.parse.quote_plus(item_source.name)}/view/"
+    url = flask_util.view_url(
-    new_annotation = f"<a class='new' href='{subpath}{item_source.get_annotations_subpath(item)}'>new</a>"
+        item_source.get_annotations_subpath(item), item_source.name
+    )
+    new_annotation = f"<a class='new' href='{url}'>new</a>"
     annotations = (
         ", ".join(
             [
-                f"<a href='{subpath}{a.descriptor}/'>{a.name}</a>"
+                f"<a href='{CacheKey(item, item_source, a).view_url}/'>{a.name}</a>"
                 for a in item.annotations
             ]
         )
@@ -25,13 +31,11 @@ def render_annotations(item, item_source):
         if item.annotations
         else ""
     )
-
     return " | annotations: " + annotations + new_annotation
 
 
 def render_item(item, item_source):
-    url = f"/source/{urllib.parse.quote_plus(item_source.name)}/view/{item.descriptor}/"
+    item_string = f"<li> <a href='{ CacheKey(item, item_source).view_url }/'>{item.name}</a> {render_annotations(item, item_source)}</li>"
-    item_string = f"<li> <a href='{ url }'>{item.name}</a> {render_annotations(item, item_source)}</li>"
     return item_string
 
 
@@ -42,9 +46,7 @@ def render_item_tree(item_tree, item_source):
         else ""
     )
     branches = (
-        "\n".join(
+        "\n".join([render_item_tree(b, item_source) for b in item_tree.branches])
-            [render_item_tree(b, item_source) for b in item_tree.branches]
-        )
         if item_tree.branches
         else ""
     )
@@ -52,11 +54,7 @@ def render_item_tree(item_tree, item_source):
     if item_tree.descriptor:
         descriptor = item_tree.descriptor.lstrip("/")
         url = f"/filecrawl/{descriptor}?source={item_source.name}"
-        name = (
+        name = descriptor.rsplit("/")[1] if descriptor.find("/") >= 0 else descriptor
-            descriptor.rsplit("/")[1]
-            if descriptor.find("/") >= 0
-            else descriptor
-        )
         return f"<li><a href='{url}'>{name}</a>{html}</li>"
     else:
         return html
@@ -64,5 +62,6 @@ def render_item_tree(item_tree, item_source):
 
 def render_item_source(item_source, filter=None):
     item_tree = item_source.list_items(filter)
-    heading = f"<h6><a href='/filecrawl.html?source={urllib.parse.quote_plus(item_source.name)}'>{item_source.name}</a></h6>"
+    filterpart = "" if filter is None else ":" + filter
+    heading = f"<h6><a href='/filecrawl.html?source={urllib.parse.quote_plus(item_source.name)}'>{item_source.name}</a>{filterpart}</h6>"
     return heading + render_item_tree(item_tree, item_source)

cellxgene_gateway/flask_util.py

@@ -7,9 +7,27 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
-from flask import request
+from flask import request, url_for
 
 
 def querystring():
     qs = request.query_string.decode()
     return f"?{qs}" if len(qs) > 0 else ""
+
+
+include_source_in_url = False
+
+
+def url(endpoint, descriptor, source_name):
+    if include_source_in_url:
+        return url_for(endpoint, source_name=source_name, path=descriptor)
+    else:
+        return url_for(endpoint, path=descriptor)
+
+
+def view_url(descriptor, source_name):
+    return url("do_view", descriptor, source_name)
+
+
+def relaunch_url(descriptor, source_name):
+    return url("do_relaunch", descriptor, source_name)

cellxgene_gateway/gateway.py

@@ -23,19 +23,19 @@ from flask import (
     url_for,
 )
 from flask_api import status
+from werkzeug.middleware.proxy_fix import ProxyFix
 from werkzeug.utils import secure_filename
 
-from cellxgene_gateway import env
+from cellxgene_gateway import env, flask_util
 from cellxgene_gateway.backend_cache import BackendCache
 from cellxgene_gateway.cache_entry import CacheEntryStatus
+from cellxgene_gateway.cache_key import CacheKey
 from cellxgene_gateway.cellxgene_exception import CellxgeneException
-from cellxgene_gateway.dir_util import create_dir, is_subdir
 from cellxgene_gateway.extra_scripts import get_extra_scripts
 from cellxgene_gateway.filecrawl import render_item_source
 from cellxgene_gateway.process_exception import ProcessException
 from cellxgene_gateway.prune_process_cache import PruneProcessCache
 from cellxgene_gateway.util import current_time_stamp
-from cellxgene_gateway.cache_key import CacheKey
 
 app = Flask(__name__)
 
@@ -45,6 +45,7 @@ default_item_source = None
 
 def _force_https(app):
     def wrapper(environ, start_response):
+        if env.external_protocol is not None:
             environ["wsgi.url_scheme"] = env.external_protocol
         return app(environ, start_response)
 
@@ -52,9 +53,23 @@ def _force_https(app):
 
 
 app.wsgi_app = _force_https(app.wsgi_app)
+if (
+    env.proxy_fix_for > 0
+    or env.proxy_fix_proto > 0
+    or env.proxy_fix_host > 0
+    or env.proxy_fix_port > 0
+    or env.proxy_fix_prefix > 0
+):
+    app.wsgi_app = ProxyFix(
+        app.wsgi_app,
+        x_for=env.proxy_fix_for,
+        x_proto=env.proxy_fix_proto,
+        x_host=env.proxy_fix_host,
+        x_port=env.proxy_fix_port,
+        x_prefix=env.proxy_fix_prefix,
+    )
 
 cache = BackendCache()
-location = f"{env.external_protocol}://{env.external_host}"
 
 
 @app.errorhandler(CellxgeneException)
@@ -90,7 +105,7 @@ def handle_invalid_process(error):
             http_status=error.http_status,
             stdout=error.stdout,
             stderr=error.stderr,
-            dataset=error.key.h5ad_item.descriptor,
+            relaunch_url=error.key.relaunch_url(),
             annotation_file=error.key.annotation_descriptor,
         ),
         error.http_status,
@@ -201,7 +216,11 @@ def do_view(path, source_name=None):
 
 @app.route("/cache_status", methods=["GET"])
 def do_GET_status():
-    return render_template("cache_status.html", entry_list=cache.entry_list)
+    return render_template(
+        "cache_status.html",
+        entry_list=cache.entry_list,
+        extra_scripts=get_extra_scripts(),
+    )
 
 
 @app.route("/cache_status.json", methods=["GET"])
@@ -225,15 +244,14 @@ def do_GET_status_json():
 
 @app.route("/relaunch/<path:path>", methods=["GET"])
 def do_relaunch(path):
-    source_name = request.args.get("source") or default_item_source.name
+    source_name = request.args.get("source_name") or default_item_source.name
     source = matching_source(source_name)
     key = CacheKey.for_lookup(source, source.lookup(path))
     match = cache.check_entry(key)
     if not match is None:
         match.terminate()
-    qs = request.query_string.decode()
     return redirect(
-        url_for("do_view", path=path) + (f"?{qs}" if len(qs) > 0 else ""),
+        key.view_url,
         code=302,
     )
 
@@ -287,6 +305,7 @@ def main():
         default_item_source = "local"
     if len(item_sources) == 0:
         raise Exception("Please specify CELLXGENE_DATA or CELLXGENE_BUCKET")
+    flask_util.include_source_in_url = len(item_sources) > 1
 
     launch()
 

cellxgene_gateway/items/file/fileitem.py

@@ -18,10 +18,11 @@ class FileItem(Item):
     The Item superclass expects a 'name' and 'type'.
     """
 
-    def __init__(self, subpath: str, *args, **kwargs):
+    def __init__(self, subpath: str, ext: str = "", *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.subpath = subpath
+        self.ext = ext
 
     @property
     def descriptor(self) -> str:
-        return os.path.join(self.subpath, self.name).strip("/")
+        return os.path.join(self.subpath, self.name + self.ext).strip("/")

cellxgene_gateway/items/file/fileitem_source.py

@@ -51,7 +51,7 @@ class FileItemSource(ItemSource):
         return self.convert_h5ad_path_to_annotation(item.descriptor)
 
     def list_items(self, filter: str = None) -> ItemTree:
-        item_tree = self.scan_directory()
+        item_tree = self.scan_directory("" if filter is None else filter)
 
         """def get_items(dir):
             if dir.branches:
@@ -67,9 +67,7 @@ class FileItemSource(ItemSource):
         base_path = os.path.join(self.base_path, subpath)
 
         if not os.path.exists(base_path):
-            raise Exception(
+            raise Exception(f"Path for local files '{base_path}' does not exist.")
-                f"Path for local files '{base_path}' does not exist."
-            )
 
         filepath_map = dict(
             (filepath, os.path.join(base_path, filepath))
@@ -95,14 +93,12 @@ class FileItemSource(ItemSource):
         ]
 
         items = [
-            self.make_fileitem_from_path(filename, subpath)
+            self.make_fileitem_from_path(filename, subpath) for filename in h5ad_paths
-            for filename in h5ad_paths
         ]
         branches = None
         if len(subdirs) > 0:
             branches = [
-                self.scan_directory(os.path.join(subpath, subdir))
+                self.scan_directory(os.path.join(subpath, subdir)) for subdir in subdirs
-                for subdir in subdirs
             ]
 
         return ItemTree(subpath, items, branches)
@@ -128,14 +124,13 @@ class FileItemSource(ItemSource):
     def lookup(self, indescriptor: str) -> LookupResult:
         descriptor = indescriptor.strip("/")
         if descriptor.endswith(self.annotation_file_suffix):
-            annotation_item = self.shallowitem_from_descriptor(
+            annotation_item = self.shallowitem_from_descriptor(descriptor, True)
-                descriptor, True
-            )
             h5ad_descriptor = self.convert_annotation_path_to_h5ad(
                 annotation_item.subpath
             )
             item = self.lookup_item(h5ad_descriptor)
             if item is not None:
+                dir_util.ensure_dir_exists(self.full_path(annotation_item.subpath))
                 return LookupResult(item, annotation_item)
         else:
             item = self.lookup_item(descriptor)
@@ -155,9 +150,16 @@ class FileItemSource(ItemSource):
     def make_fileitem_from_path(
         self, filename, subpath, is_annotation=False, is_shallow=False
     ) -> FileItem:
+        if is_annotation and filename.endswith(self.annotation_file_suffix):
+            name = filename[: -len(self.annotation_file_suffix)]
+            ext = self.annotation_file_suffix
+        else:
+            name = filename
+            ext = ""
         item = FileItem(
             subpath=subpath,
-            name=filename,
+            name=name,
+            ext=ext,
             type=ItemType.annotation if is_annotation else ItemType.h5ad,
         )
 
@@ -172,14 +174,10 @@ class FileItemSource(ItemSource):
         annotations_fullpath = self.full_path(annotations_subpath)
         if os.path.isdir(annotations_fullpath):
             return [
-                self.make_fileitem_from_path(
+                self.make_fileitem_from_path(annotation, annotations_subpath, True)
-                    annotation, annotations_subpath, True
-                )
                 for annotation in sorted(os.listdir(annotations_fullpath))
                 if annotation.endswith(self.annotation_file_suffix)
-                and os.path.isfile(
+                and os.path.isfile(os.path.join(annotations_fullpath, annotation))
-                    os.path.join(annotations_fullpath, annotation)
-                )
             ]
         else:
             return None

cellxgene_gateway/items/item.py

@@ -18,9 +18,7 @@ class ItemType(Enum):
 
 
 class Item(ABC):
-    def __init__(
+    def __init__(self, name: str, type: ItemType, annotations: List["Item"] = None):
-        self, name: str, type: ItemType, annotations: List["Item"] = None
-    ):
         self.name = name
         self.type = type
         self.annotations = annotations

cellxgene_gateway/items/s3/s3item_source.py

@@ -7,13 +7,15 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
+from os.path import basename, dirname, join
 from typing import List
-from os.path import join, dirname, basename
+
-from cellxgene_gateway import dir_util
 import s3fs
-from cellxgene_gateway.items.s3.s3item import S3Item
+
+from cellxgene_gateway import dir_util
 from cellxgene_gateway.items.item import ItemTree, ItemType
 from cellxgene_gateway.items.item_source import ItemSource, LookupResult
+from cellxgene_gateway.items.s3.s3item import S3Item
 
 
 class S3ItemSource(ItemSource):
@@ -27,13 +29,20 @@ class S3ItemSource(ItemSource):
     ):
         self._name = name
         self.s3 = s3fs.S3FileSystem()
+        if bucket.startswith("s3://"):
+            raise Exception(
+                f"Bucket name should not include s3:// prefix, got {bucket}"
+            )
         self.bucket = bucket
         self.h5ad_suffix = h5ad_suffix
         self.annotation_dir_suffix = annotation_dir_suffix
         self.annotation_file_suffix = annotation_file_suffix
 
-    def url(self, path):
+    def url(self, key):
-        return "s3://" + join(self.bucket, path)
+        return "s3://" + self.bucket + "/" + key
+
+    def remove_bucket(self, filepath):
+        return filepath[len(self.bucket) :].lstrip("/")
 
     @property
     def name(self):
@@ -55,51 +64,44 @@ class S3ItemSource(ItemSource):
         return self.convert_h5ad_key_to_annotation(item.descriptor)
 
     def list_items(self, filter: str = None) -> ItemTree:
-        item_tree = self.scan_directory()
+        item_tree = self.scan_directory("" if filter is None else filter)
         return item_tree
 
-    def scan_directory(self, subpath="") -> dict:
+    def scan_directory(self, directory_key="") -> dict:
-        url = self.url(subpath)
+        url = self.url(directory_key)
 
         if not self.s3.exists(url):
             raise Exception(f"S3 url '{url}' does not exist.")
 
         s3key_map = dict(
-            (filepath[len(self.bucket) :], "s3://" + filepath)
+            (self.remove_bucket(filepath), "s3://" + filepath)
             for filepath in sorted(self.s3.ls(url))
         )
 
         def is_annotation_dir(dir_s3key):
             return (
                 dir_s3key.endswith(self.annotation_dir_suffix)
-                and self.convert_annotation_key_to_h5ad(dir_s3key)
+                and self.convert_annotation_key_to_h5ad(dir_s3key) in h5ad_keys
-                in h5ad_paths
             )
 
-        h5ad_paths = [
+        h5ad_keys = [
             filepath
             for filepath, item_url in s3key_map.items()
             if self.is_h5ad_url(item_url)
         ]
 
-        subdirs = [
+        subdir_keys = [
             filepath
             for filepath, item_url in s3key_map.items()
             if self.s3.isdir(item_url) and not is_annotation_dir(filepath)
         ]
 
-        items = [
+        items = [self.make_s3item_from_key(basename(key), key) for key in h5ad_keys]
-            self.make_s3item_from_key(filename, join(subpath, filename))
-            for filename in h5ad_paths
-        ]
         branches = None
-        if len(subdirs) > 0:
+        if len(subdir_keys) > 0:
-            branches = [
+            branches = [self.scan_directory(key) for key in subdir_keys]
-                self.scan_directory(join(subpath, subdir))
-                for subdir in subdirs
-            ]
 
-        return ItemTree(subpath, items, branches)
+        return ItemTree(directory_key, items, branches)
 
     def create_annotation(self, item: S3Item, name: str) -> S3Item:
         annotation = self.make_s3item_from_key(
@@ -119,9 +121,7 @@ class S3ItemSource(ItemSource):
     def lookup(self, indescriptor: str) -> LookupResult:
         descriptor = indescriptor.strip("/")
         if descriptor.endswith(self.annotation_file_suffix):
-            annotation_item = self.shallowitem_from_descriptor(
+            annotation_item = self.shallowitem_from_descriptor(descriptor, True)
-                descriptor, True
-            )
             if not self.s3.exists(self.url(annotation_item.s3key)):
                 with self.s3.open(self.url(annotation_item.s3key), "w") as f:
                     f.write("")
@@ -161,11 +161,11 @@ class S3ItemSource(ItemSource):
         if self.s3.isdir(annotations_fullpath):
             return [
                 self.make_s3item_from_key(
-                    annotation, join(annotations_subpath, annotation), True
+                    basename(annotation), self.remove_bucket(annotation), True
                 )
                 for annotation in sorted(self.s3.ls(annotations_fullpath))
                 if annotation.endswith(self.annotation_file_suffix)
-                and self.s3.isfile(join(annotations_fullpath, annotation))
+                and self.s3.isfile("s3://" + annotation)
             ]
         else:
             return None

cellxgene_gateway/path_util.py

@@ -1,41 +0,0 @@
-# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
-# under the Apache License, Version 2.0 (the "License"); you may not use
-# this file except in compliance with the License. You may obtain a copy
-# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
-# required by applicable law or agreed to in writing, software distributed
-# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
-# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
-# the specific language governing permissions and limitations under the License.
-
-import os
-
-from flask_api import status
-
-from cellxgene_gateway.cache_key import CacheKey
-from cellxgene_gateway.cellxgene_exception import CellxgeneException
-from cellxgene_gateway.dir_util import make_h5ad
-
-
-def validate_exists(file_path):
-    if not os.path.exists(file_path):
-        raise CellxgeneException(
-            "File does not exist: " + file_path, status.HTTP_400_BAD_REQUEST
-        )
-
-
-def validate_is_file(file_path):
-    validate_exists(file_path)
-    if not os.path.isfile(file_path):
-        raise CellxgeneException(
-            "Path is not file: " + file_path, status.HTTP_400_BAD_REQUEST
-        )
-    return
-
-
-def validate_is_dir(file_path):
-    validate_exists(file_path)
-    if not os.path.isdir(file_path):
-        raise CellxgeneException(
-            "Path is not dir: " + file_path, status.HTTP_400_BAD_REQUEST
-        )
-    return

cellxgene_gateway/prune_process_cache.py

@@ -10,14 +10,15 @@
 import logging
 import time
 
-from cellxgene_gateway.env import ttl
+from cellxgene_gateway import env, util
-from cellxgene_gateway.util import current_time_stamp
+
+logger = logging.getLogger(__name__)
 
 
 class PruneProcessCache:
     def __init__(self, cache):
         self.cache = cache
-        self.expire_seconds = 3600 if ttl is None else int(ttl)
+        self.expire_seconds = 3600 if env.ttl is None else int(env.ttl)
 
     def __call__(self):
         while True:
@@ -25,26 +26,22 @@ class PruneProcessCache:
             self.prune()
 
     def prune(self):
-        timestamp = current_time_stamp()
+        timestamp = util.current_time_stamp()
         cutoff = timestamp - self.expire_seconds
-        processes_to_delete = [
+        processes_to_delete = [p for p in self.cache.entry_list if p.timestamp < cutoff]
-            p for p in self.cache.entry_list if p.timestamp < cutoff
-        ]
         processes_to_keep = [
             p for p in self.cache.entry_list if not p.timestamp < cutoff
         ]
-        logger = logging.getLogger("cellxgene_gateway")
+
         logger.debug(
-            f"Cutoff {cutoff} = timestamp {timestamp} - expire seconds {self.expire_seconds} , keeping {processes_to_keep}"
+            f"Cutoff {cutoff} = timestamp {timestamp} - expire seconds {self.expire_seconds} , keeping {processes_to_keep}, pruning {processes_to_delete}"
         )
 
         for process in processes_to_delete:
             try:
-                logger.info(
+                logger.info(f"pruning process {process.pid} ({process.key.descriptor})")
-                    f"pruning process {process.pid} ({process.key.dataset})"
-                )
                 self.cache.prune(process)
             except Exception:
                 logger.exception(
-                    "failed to prune process {process.pid} ({process.dataset})"
+                    "failed to prune process {process.pid} ({process.key.descriptor})"
                 )

cellxgene_gateway/static/js/annotation.js

@@ -1,5 +1,5 @@
 // neandertal javascript
-// TODO: rewrite this --
+// Annotations only work with file itemsources at the moment. If they work with others in the future we may need to revisit this.
 const new_annotation_callback = (() =>{
     const suffix = `.csv`;
     return (e) => {

cellxgene_gateway/subprocess_backend.py

@@ -14,11 +14,7 @@ from flask_api import status
 
 from cellxgene_gateway.cache_entry import CacheEntryStatus
 from cellxgene_gateway.dir_util import make_annotations
-from cellxgene_gateway.env import (
+from cellxgene_gateway.env import cellxgene_args, enable_annotations, enable_backed_mode
-    enable_annotations,
-    enable_backed_mode,
-    cellxgene_args,
-)
 from cellxgene_gateway.process_exception import ProcessException
 
 
@@ -26,14 +22,10 @@ class SubprocessBackend:
     def __init__(self):
         pass
 
-    def create_cmd(
+    def create_cmd(self, cellxgene_loc, file_path, port, scripts, annotation_file_path):
-        self, cellxgene_loc, file_path, port, scripts, annotation_file_path
-    ):
         if enable_annotations and not annotation_file_path is None:
             if annotation_file_path == "":
-                extra_args = (
+                extra_args = f" --annotations-dir {make_annotations(file_path)}"
-                    f" --annotations-dir {make_annotations(file_path)}"
-                )
             else:
                 extra_args = f" --annotations-file {annotation_file_path}"
         else:

cellxgene_gateway/templates/cache_status.html

@@ -48,7 +48,7 @@
                <tr>
                     <td>{{ entry.pid }}</td>
                     <td><a
-                              href="{{ url_for('do_view', path=entry.key.descriptor, source_name=entry.key.source_name) }}">{{ entry.key.h5ad_item.descriptor }}</a>
+                              href="{{ entry.key.view_url }}">{{ entry.key.h5ad_item.descriptor }}</a>
                     </td>
                     <td>{{ entry.key.annotation_descriptor }}</td>
                     <td>{{ entry.source_name }}</td>

cellxgene_gateway/templates/cellxgene_error.html

@@ -28,11 +28,11 @@
           
      <h4>{{ message }}</h4>
 
-     <a href="/filecrawl.html">
+     <a href="{{ url_for('filecrawl') }}">
           Please click here to be redirected to the file directory.
      </a>
      <br>
-     <a href="/">
+     <a href="{{ url_for('index') }}">
           Please click here to return to the homepage.
      </a>
      </div>

cellxgene_gateway/templates/filecrawl.html

@@ -36,10 +36,10 @@
           Navigation:
      <ul>
      {% if path %}
-     <li><a href="/filecrawl.html">top level</a></li>
+     <li><a href="{{ url_for('filecrawl') }}">top level</a></li>
      {% else %}
      {% endif %}
-     <li><a href="/">homepage</a></li>
+     <li><a href="{{ url_for('index') }}">homepage</a></li>
      </ul>
      </p>
      <script>

cellxgene_gateway/templates/index.html

@@ -35,12 +35,12 @@
 		Links:	
 	</h1>
 	<div class="list-group" style="width:50%;padding-left:65px">
-	  <a href="/filecrawl.html" class="list-group-item list-group-item-action">
+	  <a href="{{ url_for('filecrawl') }}" class="list-group-item list-group-item-action">
 		  <u>File Crawler: Allows you to view all uploaded data.</u></a>
 		  
 	</div>
 	<div class="list-group" style="width:50%;padding-left:65px">
-		<a href="/cache_status" class="list-group-item list-group-item-action">
+		<a href="{{ url_for('do_GET_status') }}" class="list-group-item list-group-item-action">
 			<u>Cache Status: view status of launched cellxgene servers.</u></a>
 	</div>
 	

cellxgene_gateway/templates/loading.html

@@ -35,11 +35,11 @@
           The page will refresh shortly.
      </p>
 
-     <a href="/filecrawl.html">
+     <a href="{{ url_for('filecrawl') }}">
           Please click here to be redirected to the file directory.
      </a>
      <br>
-     <a href="/">
+     <a href="{{ url_for('index') }}">
           Please click here to return to the homepage.
      </a>
      </div>

cellxgene_gateway/templates/process_error.html

@@ -36,13 +36,13 @@
 <h4>Options</h4>
      Please choose one of the following, or use the back button:
      <ul>
-          <li><a href="{{url_for('do_relaunch', path=dataset)}}">
+          <li><a href="{{ relaunch_url }}">
                     Attempt to relaunch the cellxgene server.
                </a></li>
-          <li><a href="/filecrawl.html">
+          <li><a href="{{ url_for('filecrawl') }}">
                Return to the file directory.
           </a></li>
-          <li><a href="/">
+          <li><a href="{{ url_for('index') }}">
                Return to the homepage.
           </a></li>
      </ul>

environment.yml

@@ -7,6 +7,7 @@ dependencies:
 - flask
 - psutil
 - black
+- coverage
 - pip
 - pip:
   - flask-api

tests/items/file/test_fileitem_source.py

@@ -0,0 +1,43 @@
+import tempfile
+import unittest
+from unittest.mock import patch
+
+from cellxgene_gateway.items.file.fileitem_source import FileItemSource
+
+
+def stub_join(path):
+    path.join = lambda x, y: x + "/" + y
+
+
+class TestFileItemSource(unittest.TestCase):
+    @patch("os.path")
+    @patch("os.listdir")
+    def test_list_items_GIVEN_no_subpath_THEN_checks_dir(self, listdir, path):
+        stub_join(path)
+        source = FileItemSource("/tmp/unittest", "local")
+        source.list_items()
+        path.exists.assert_called_once_with("/tmp/unittest/")
+
+    @patch("os.path")
+    @patch("os.listdir")
+    def test_list_items_GIVEN_subpath_THEN_checks_subpath(self, listdir, path):
+        stub_join(path)
+        source = FileItemSource("/tmp/unittest", "local")
+        source.list_items("foo")
+        path.exists.assert_called_once_with("/tmp/unittest/foo")
+
+    def test_make_fileitem_from_path_GIVEN_annotation_file_THEN_name_lacks_csv(
+        self,
+    ):
+        source = FileItemSource(tempfile.gettempdir(), "local")
+        item = source.make_fileitem_from_path(
+            "customanno.csv", "someh5ad_annotations", True
+        )
+        self.assertEqual(item.name, "customanno")
+        self.assertEqual(item.descriptor, "someh5ad_annotations/customanno.csv")
+
+    def test_make_fileitem_from_path_GIVEN_h5ad_file_THEN_returns_name(self):
+        source = FileItemSource(tempfile.gettempdir(), "local")
+        item = source.make_fileitem_from_path("someanalysis.h5ad", "studydir")
+        self.assertEqual(item.name, "someanalysis.h5ad")
+        self.assertEqual(item.descriptor, "studydir/someanalysis.h5ad")

tests/items/s3/test_s3item_source.py

@@ -0,0 +1,169 @@
+import unittest
+from unittest.mock import MagicMock, Mock, patch
+
+from cellxgene_gateway.items.item import ItemType
+from cellxgene_gateway.items.s3.s3item import S3Item
+from cellxgene_gateway.items.s3.s3item_source import S3ItemSource
+
+
+class TestScanDirectory(unittest.TestCase):
+    @patch("s3fs.S3FileSystem")
+    def test_GIVEN_invalid_bucket_THEN_throws_error(self, s3func):
+        class S3Mock:
+            def exists(path):
+                if path in ["s3://my-bucket/"]:
+                    return False
+
+        s3func.return_value = S3Mock
+        source = S3ItemSource("my-bucket")
+        with self.assertRaises(Exception) as context:
+            source.scan_directory()
+        self.assertEqual(
+            "S3 url 's3://my-bucket/' does not exist.",
+            str(context.exception),
+        )
+
+    @patch("s3fs.S3FileSystem")
+    def test__GIVEN_multilevel_bucket_THEN_properly_recurses_suburls(self, s3func):
+        class S3Mock:
+            def exists(path):
+                if path in [
+                    "s3://my-bucket/",
+                    "s3://my-bucket/pbmc3k.h5ad",
+                    "s3://my-bucket/lvl1",
+                    "s3://my-bucket/lvl1/pbmc3k_l1.h5ad",
+                    "s3://my-bucket/lvl1/lvl2",
+                    "s3://my-bucket/lvl1/lvl2/pbmc3k_l2.h5ad",
+                ]:
+                    return True
+                raise Exception("exists called with " + path)
+
+            def ls(path):
+                if path == "s3://my-bucket/":
+                    return [
+                        "my-bucket/lvl1",
+                        "my-bucket/pbmc3k.h5ad",
+                        "my-bucket/pbmc3k_annotations",
+                    ]
+                elif path == "s3://my-bucket/pbmc3k_annotations":
+                    return ["my-bucket/pbmc3k_annotations/annot.csv"]
+                elif path == "s3://my-bucket/lvl1":
+                    return ["my-bucket/lvl1/lvl2", "my-bucket/lvl1/pbmc3k_l1.h5ad"]
+                elif path == "s3://my-bucket/lvl1/lvl2":
+                    return ["my-bucket/lvl1/lvl2/pbmc3k_l2.h5ad"]
+
+                raise Exception("ls called with " + path)
+
+            def isdir(path):
+                if path in [
+                    "s3://my-bucket/lvl1",
+                    "s3://my-bucket/pbmc3k_annotations",
+                    "s3://my-bucket/lvl1/lvl2",
+                ]:
+                    return True
+                if path in [
+                    "s3://my-bucket/pbmc3k.h5ad",
+                    "s3://my-bucket/lvl1/pbmc3k_l1.h5ad",
+                    "s3://my-bucket/lvl1/pbmc3k_l1_annotations",
+                    "s3://my-bucket/lvl1/lvl2/pbmc3k_l2.h5ad",
+                    "s3://my-bucket/lvl1/lvl2/pbmc3k_l2_annotations",
+                ]:
+                    return False
+                raise Exception("isdir called with " + path)
+
+            def isfile(path):
+                if path in ["s3://my-bucket/pbmc3k_annotations/annot.csv"]:
+                    return True
+                if path in ["s3://my-bucket/pbmc3k_annotations"]:
+                    return False
+                raise Exception("isfile called with " + path)
+
+        s3func.return_value = S3Mock
+        source = S3ItemSource("my-bucket")
+        tree = source.scan_directory()
+
+        def s3item_compare(i1, i2, msg=""):
+            self.assertEqual(i1.name, i2.name, "name equals")
+            self.assertEqual(i1.type, i2.type, "type equals")
+            self.assertEqual(i1.s3key, i2.s3key, "s3key equals")
+            if i1.annotations is None:
+                self.assertEqual(i1.annotations, i2.annotations, "annotations equals")
+            else:
+                self.assertEqual(
+                    len(i1.annotations),
+                    len(i2.annotations),
+                    "annotations length equals",
+                )
+                for a1, a2 in zip(i1.annotations, i2.annotations):
+                    self.assertEqual(a1, a2)
+            return True
+
+        self.addTypeEqualityFunc(S3Item, s3item_compare)
+
+        def assertTree(t, descriptor, items):
+            self.assertEqual(t.descriptor, descriptor)
+            self.assertEqual(len(t.items), len(items))
+            for i1, i2 in zip(t.items, items):
+                self.assertEqual(i1, i2)
+
+        assertTree(
+            tree,
+            "",
+            [
+                S3Item(
+                    "pbmc3k.h5ad",
+                    name="pbmc3k.h5ad",
+                    type=ItemType.h5ad,
+                    annotations=[
+                        S3Item(
+                            "pbmc3k_annotations/annot.csv",
+                            name="annot.csv",
+                            type=ItemType.annotation,
+                        )
+                    ],
+                )
+            ],
+        )
+        self.assertEqual(len(tree.branches), 1)
+        lvl1 = tree.branches[0]
+        assertTree(
+            lvl1,
+            "lvl1",
+            [
+                S3Item(
+                    "lvl1/pbmc3k_l1.h5ad",
+                    name="pbmc3k_l1.h5ad",
+                    type=ItemType.h5ad,
+                    annotations=None,
+                )
+            ],
+        )
+        self.assertEqual(len(lvl1.branches), 1)
+        lvl2 = lvl1.branches[0]
+        assertTree(
+            lvl2,
+            "lvl1/lvl2",
+            [
+                S3Item(
+                    "lvl1/lvl2/pbmc3k_l2.h5ad",
+                    name="pbmc3k_l2.h5ad",
+                    type=ItemType.h5ad,
+                    annotations=None,
+                )
+            ],
+        )
+        self.assertEqual(lvl2.branches, None)
+
+
+class TestListItems(unittest.TestCase):
+    def test_GIVEN_filter_THEN_pass_filter_into_scan_directory(self):
+        source = S3ItemSource("my-bucket")
+        source.scan_directory = MagicMock()
+        tree = source.list_items("some-filter")
+        source.scan_directory.assert_called_once_with("some-filter")
+
+    def test_GIVEN_no_filter_THEN_pass_empty_string_into_scan_directory(self):
+        source = S3ItemSource("my-bucket")
+        source.scan_directory = MagicMock()
+        tree = source.list_items()
+        source.scan_directory.assert_called_once_with("")

tests/test_backend_cache.py

@@ -0,0 +1,28 @@
+import unittest
+from unittest.mock import MagicMock, patch
+
+from cellxgene_gateway.backend_cache import is_port_in_use
+
+
+class TestIsPortInUse(unittest.TestCase):
+    @patch("socket.socket")
+    def test_GIVEN_free_port_THEN_returns_true(self, socketMock):
+        connectMock = socketMock()
+        connectMock.connect_ex.return_value = 0
+        connectMock.__enter__.return_value = connectMock
+        self.assertEqual(is_port_in_use(123), True)
+        self.assertTrue(connectMock.__enter__.calledOnce)
+        self.assertTrue(connectMock.__exit__.calledOnce)
+        self.assertTrue(connectMock.connect_ex.calledOnceWith("a"))
+        self.assertTrue(socketMock.calledOnceWith("a"))
+
+    @patch("socket.socket")
+    def test_GIVEN_used_port_THEN_returns_false(self, socketMock):
+        connectMock = socketMock()
+        connectMock.__enter__.return_value = connectMock
+        connectMock.connect_ex.return_value = 1
+        self.assertTrue(connectMock.__enter__.calledOnce)
+        self.assertTrue(connectMock.__exit__.calledOnce)
+        self.assertTrue(connectMock.connect_ex.calledOnceWith("a"))
+        self.assertTrue(socketMock.calledOnceWith("a"))
+        self.assertEqual(is_port_in_use(123), False)

tests/test_cache_entry.py

@@ -1,33 +1,62 @@
 import unittest
+
+from flask import Flask
+
+from cellxgene_gateway import flask_util
 from cellxgene_gateway.cache_entry import CacheEntry, CacheEntryStatus
 from cellxgene_gateway.cache_key import CacheKey
-from cellxgene_gateway.items.item import ItemType
+from cellxgene_gateway.gateway import app
 from cellxgene_gateway.items.file.fileitem import FileItem
 from cellxgene_gateway.items.file.fileitem_source import FileItemSource
+from cellxgene_gateway.items.item import ItemType
 
 key = CacheKey(
-    FileItem("/czi/", "pbmc3k.h5ad", ItemType.h5ad),
+    FileItem("/czi/", name="pbmc3k.h5ad", type=ItemType.h5ad),
     FileItemSource("/tmp", "local"),
 )
 
 
 class TestRenderEntry(unittest.TestCase):
+    def setUp(self):
+        self.app = app
+        self.app_context = self.app.test_request_context()
+        self.app_context.push()
+        self.client = self.app.test_client()
+
     def test_GIVEN_key_and_port_THEN_returns_loading_CacheEntry(self):
         entry = CacheEntry.for_key("some-key", 1)
         self.assertEqual(entry.status, CacheEntryStatus.loading)
 
     def test_GIVEN_absolute_static_url_THEN_include_path(self):
+        flask_util.include_source_in_url = False
         actual = CacheEntry.for_key(key, 8000).rewrite_text_content(
             "src:url(/static/assets/"
         )
-        expected = "src:url(http://localhost:5005/source/local/view/czi/pbmc3k.h5ad/static/assets/"
+        expected = "src:url(/view/czi/pbmc3k.h5ad/static/assets/"
         self.assertEqual(actual, expected)
 
     def test_GIVEN_absolute_src_THEN_include_path(self):
+        flask_util.include_source_in_url = False
         actual = CacheEntry.for_key(key, 8000).rewrite_text_content(
             '<link rel="shortcut icon" href="/static/assets/favicon.ico">'
         )
-        expected = '<link rel="shortcut icon" href="http://localhost:5005/source/local/view/czi/pbmc3k.h5ad/static/assets/favicon.ico">'
+        expected = '<link rel="shortcut icon" href="/view/czi/pbmc3k.h5ad/static/assets/favicon.ico">'
+        self.assertEqual(actual, expected)
+
+    def test_GIVEN_absolute_static_url_include_source_THEN_include_path(self):
+        flask_util.include_source_in_url = True
+        actual = CacheEntry.for_key(key, 8000).rewrite_text_content(
+            "src:url(/static/assets/"
+        )
+        expected = "src:url(/source/local/view/czi/pbmc3k.h5ad/static/assets/"
+        self.assertEqual(actual, expected)
+
+    def test_GIVEN_absolute_src_include_source_THEN_include_path(self):
+        flask_util.include_source_in_url = True
+        actual = CacheEntry.for_key(key, 8000).rewrite_text_content(
+            '<link rel="shortcut icon" href="/static/assets/favicon.ico">'
+        )
+        expected = '<link rel="shortcut icon" href="/source/local/view/czi/pbmc3k.h5ad/static/assets/favicon.ico">'
         self.assertEqual(actual, expected)
 
 

tests/test_dir_util.py

@@ -0,0 +1,35 @@
+import unittest
+from unittest.mock import MagicMock, patch
+
+from cellxgene_gateway.dir_util import ensure_dir_exists, make_annotations, make_h5ad
+
+
+class TestMakeH5ad(unittest.TestCase):
+    def test_GIVEN_annotation_dir_THEN_returns_h5ad(self):
+        self.assertEqual(make_h5ad("pbmc_annotations"), "pbmc.h5ad")
+
+
+class TestMakeAnnotations(unittest.TestCase):
+    def test_GIVEN_h5ad_THEN_returns_annotations(self):
+        self.assertEqual(make_annotations("pbmc.h5ad"), "pbmc_annotations")
+
+
+class TestMakeAnnotations(unittest.TestCase):
+    def test_GIVEN_h5ad_THEN_returns_annotations(self):
+        self.assertEqual(make_annotations("pbmc.h5ad"), "pbmc_annotations")
+
+
+class TestEnsureDirExists(unittest.TestCase):
+    @patch("os.path.exists")
+    @patch("os.makedirs")
+    def test_GIVEN_existing_THEN_does_not_call_makedir(self, makedirsMock, existsMock):
+        existsMock.return_value = True
+        ensure_dir_exists("/foo")
+        makedirsMock.assert_not_called()
+
+    @patch("os.path.exists")
+    @patch("os.makedirs")
+    def test_GIVEN_not_existing_THEN_calls_makedir(self, makedirsMock, existsMock):
+        existsMock.return_value = False
+        ensure_dir_exists("/foo")
+        makedirsMock.assert_called_once_with("/foo")

tests/test_extra_scripts.py

@@ -21,6 +21,15 @@ class TestExtraScripts(unittest.TestCase):
     def test_GIVEN_empty_string_THEN_returns_empty_array(self):
         self.assertEqual(get_extra_scripts(), [])
 
+    @patch("cellxgene_gateway.env.extra_scripts", new="'asdf'")
+    def test_GIVEN_bare_string_THEN_throws_Exception(self):
+        with self.assertRaises(Exception) as context:
+            self.assertEqual(get_extra_scripts(), [])
+        self.assertEqual(
+            'Error parsing GATEWAY_EXTRA_SCRIPTS, expected JSON array e.g. ["https://example.com/path/to/script.js"]',
+            str(context.exception),
+        )
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_filecrawl.py

@@ -1,19 +1,17 @@
 import unittest
 from unittest.mock import MagicMock, patch
 
-from cellxgene_gateway.filecrawl import render_item
+from cellxgene_gateway.filecrawl import render_item, render_item_source
-from cellxgene_gateway.items.item import ItemType
 from cellxgene_gateway.items.file.fileitem import FileItem
 from cellxgene_gateway.items.file.fileitem_source import FileItemSource
+from cellxgene_gateway.items.item import ItemTree, ItemType
 
 source = FileItemSource("/tmp")
 
 
 class TestRenderEntry(unittest.TestCase):
     def test_GIVEN_path_both_slash_THEN_view_has_single_slash(self):
-        entry = FileItem(
+        entry = FileItem(subpath="/somepath/", name="entry", type=ItemType.h5ad)
-            subpath="/somepath/", name="entry", type=ItemType.h5ad
-        )
         rendered = render_item(entry, source)
         self.assertIn("view/somepath/entry/'", rendered)
 
@@ -31,3 +29,15 @@ class TestRenderEntry(unittest.TestCase):
         entry = FileItem(subpath="somepath", name="entry", type=ItemType.h5ad)
         rendered = render_item(entry, source)
         self.assertIn("view/somepath/entry/'", rendered)
+
+
+class TestRenderItemSource(unittest.TestCase):
+    @patch("cellxgene_gateway.items.file.fileitem_source.FileItemSource")
+    def test_GIVEN_some_filter_THEN_includes_filterpart_in_heading(self, item_source):
+        item_source.name = "FakeSource"
+        item_source.list_items.return_value = ItemTree("rootdir", [], [])
+        rendered = render_item_source(item_source, "some_filter")
+        self.assertEqual(
+            rendered,
+            "<h6><a href='/filecrawl.html?source=FakeSource'>FakeSource</a>:some_filter</h6><li><a href='/filecrawl/rootdir?source=FakeSource'>rootdir</a><ul></ul></li>",
+        )

tests/test_prune_process_cache.py

@@ -1,8 +1,16 @@
 import unittest
-from unittest.mock import MagicMock, patch
+from unittest.mock import patch, seal
 
 from cellxgene_gateway.backend_cache import BackendCache
-from cellxgene_gateway.cache_entry import CacheEntry
+from cellxgene_gateway.cache_key import CacheKey
+from cellxgene_gateway.items.file.fileitem import FileItem
+from cellxgene_gateway.items.file.fileitem_source import FileItemSource
+from cellxgene_gateway.items.item import ItemType
+
+key = CacheKey(
+    FileItem("/czi/", name="pbmc3k.h5ad", type=ItemType.h5ad),
+    FileItemSource("/tmp", "local"),
+)
 
 
 class TestPruneProcessCache(unittest.TestCase):
@@ -15,14 +23,23 @@ class TestPruneProcessCache(unittest.TestCase):
 
         cache = BackendCache()
         old.timestamp = -100
+        old.foo = 12
+        old.pid = 1
+        old.key = key
+        old.terminate.return_value = None
+        seal(old)
+        new.key = key
         cache.entry_list.append(old)
         new.timestamp = -5
+        seal(new)
         cache.entry_list.append(new)
         self.assertEqual(len(cache.entry_list), 2)
         ppc = PruneProcessCache(cache)
         ppc.prune()
         self.assertEqual(len(cache.entry_list), 1)
         self.assertEqual(cache.entry_list[0], new)
+        self.assertEqual(cache.entry_list[0], new)
+        self.assertTrue(old.terminate.called)
 
 
 if __name__ == "__main__":

tests/test_subprocess_backend.py

@@ -0,0 +1,42 @@
+import unittest
+from unittest.mock import MagicMock, patch
+
+from cellxgene_gateway.backend_cache import BackendCache
+from cellxgene_gateway.cache_entry import CacheEntry
+from cellxgene_gateway.cache_key import CacheKey
+from cellxgene_gateway.items.file.fileitem import FileItem
+from cellxgene_gateway.items.file.fileitem_source import FileItemSource
+from cellxgene_gateway.items.item import ItemType
+from cellxgene_gateway.process_exception import ProcessException
+
+
+class TestSubprocessBackend(unittest.TestCase):
+    @patch("subprocess.Popen")
+    def test_launch_GIVEN_no_stdout_THEN_throw_ProcessException(self, popen):
+        subprocess = MagicMock()
+        subprocess.stdout.readline().decode.return_value = ""
+        subprocess.stderr.read().decode.return_value = "An unexpected error"
+        popen.return_value = subprocess
+
+        key = CacheKey(
+            FileItem("/czi/", name="pbmc3k.h5ad", type=ItemType.h5ad),
+            FileItemSource("/tmp", "local"),
+        )
+        entry = CacheEntry.for_key(key, 8000)
+        from cellxgene_gateway.subprocess_backend import SubprocessBackend
+
+        backend = SubprocessBackend()
+        cellxgene_loc = "/some/cellxgene"
+        scripts = ["http://example.com/script.js", "http://example.com/script2.js"]
+
+        with self.assertRaises(ProcessException) as context:
+            backend.launch(cellxgene_loc, scripts, entry)
+        popen.assert_called_once_with(
+            [
+                "yes | /some/cellxgene launch /tmp/czi/pbmc3k.h5ad --port 8000 --host 127.0.0.1 --disable-annotations --scripts http://example.com/script.js --scripts http://example.com/script2.js"
+            ],
+            shell=True,
+            stderr=-1,
+            stdout=-1,
+        )
+        self.assertEqual("An unexpected error", context.exception.stderr)