prepare for 0.3.10 release

68 read from subprocess

.github/workflows/pr-checks.yaml

@@ -39,6 +39,7 @@ jobs:
           conda env create -f environment.yml
           eval "$(conda shell.bash hook)"
           conda activate cellxgene-gateway
+          pip install markupsafe==2.0.1 # temporary workaround for jinja2-2.11.3 calling soft_unicode in markupsafe
           python setup.py install
 
       - name: Run tests

.pre-commit-config.yaml

@@ -8,7 +8,6 @@ repos:
       types: [python]
       stages: [commit]
     - id: black
-      language_version: python3.6+
       name: black
       language: system
       entry: black

Changelog.md

@@ -1,3 +1,38 @@
+# 0.3.10
+
+* #65 Added GATEWAY_EXPIRE_SECONDS to set how long cellxgene servers can remain idle before being terminated.
+* Added GATEWAY_LOG_LEVEL to set the log level
+* #68 Close connections after reading response
+* #68 Background thread reads from output of cellxgene process until it exits
+
+# 0.3.9
+
+* Added S3_ENABLE_LISTINGS_CACHE variable (See README.md)
+
+# 0.3.8
+
+* Fixed bug #57 affecting deeply nested subdirectory listing
+
+# 0.3.7
+
+* added back /metadata/ip_address endpoint
+
+# 0.3.6
+
+* pinned version of werkzeug
+
+# 0.3.5
+
+* Pinned flask version to match cellxgene 0.17.0
+
+# 0.3.4
+
+* Fixed bug #50 affecting subdirectory listing
+
+# 0.3.3
+
+* Fixed bug #48 affecting cache pruning
+
 # 0.3.2
 
  * Fixed bug #45 affecting multi-level S3 folders

Dockerfile

@@ -0,0 +1,8 @@
+FROM python:3.9
+
+RUN pip install cellxgene-gateway 'MarkupSafe<2.1'
+
+ENV CELLXGENE_DATA=/cellxgene-data
+ENV CELLXGENE_LOCATION=/usr/local/bin/cellxgene
+
+CMD ["cellxgene-gateway"]

README.md

@@ -2,7 +2,7 @@
 
 Cellxgene Gateway allows you to use the Cellxgene Server provided by the Chan Zuckerberg Institute (https://github.com/chanzuckerberg/cellxgene) with multiple datasets. It displays an index of available h5ad (anndata) files. When a user clicks on a file name, it launches a Cellxgene Server instance that loads that particular data file and once it is available  proxies requests to that server.
 
-[![codecov](https://codecov.io/gh/Novartis/cellxgene-gateway/branch/master/graph/badge.svg?token=ndEFSzRKJn)](https://codecov.io/gh/Novartis/cellxgene-gateway)
+[![codecov](https://codecov.io/gh/Novartis/cellxgene-gateway/branch/master/graph/badge.svg?token=ndEFSzRKJn)](https://codecov.io/gh/Novartis/cellxgene-gateway) [![PyPI](https://img.shields.io/pypi/v/cellxgene-gateway)](https://pypi.org/project/cellxgene-gateway/) [![PyPI - Downloads](https://img.shields.io/pypi/dm/cellxgene-gateway)](https://pypistats.org/packages/cellxgene-gateway)
 
 # Running locally
 
@@ -73,9 +73,12 @@ Optional environment variables:
 * `EXTERNAL_PROTOCOL` - typically http when running locally, can be https when deployed if the gateway is behind a load balancer or reverse proxy that performs https termination. Default value "http"
 * `GATEWAY_IP` - ip addess of instance gateway is running on, mostly used to display SSH instructions. Defaults to `socket.gethostbyname(socket.gethostname())`
 * `GATEWAY_PORT` - local port that the gateway should bind to, defaults to 5005
+* `GATEWAY_EXPIRE_SECONDS` - time in seconds that a cellxgene process will remain idle before being terminated. Defaults to 3600 (one hour)
 * `GATEWAY_EXTRA_SCRIPTS` - JSON array of script paths, will be embedded into each page and forwarded with `--scripts` to cellxgene server
 * `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations. 
 * `GATEWAY_ENABLE_BACKED_MODE` - Set to `true` or to `1` to load AnnData in file-backed mode. This saves memory and speeds up launch time but may reduce overall performance.
+* `GATEWAY_LOG_LEVEL` - default is `INFO`. set to `DEBUG` to increase logging and to `WARNING` to decrease logging.
+* `S3_ENABLE_LISTINGS_CACHE` - Set to `true` or to `1` to cache listings of S3 folders for performance. If the cache becomes stale, set `filecrawl.html?refresh=true` query parameter to refresh the cache.
 
 If any of the following optional variables are set, [ProxyFix](https://werkzeug.palletsprojects.com/en/1.0.x/middleware/proxy_fix/) will be used.
 * `PROXY_FIX_FOR` - Number of upstream proxies setting X-Forwarded-For
@@ -86,6 +89,33 @@ If any of the following optional variables are set, [ProxyFix](https://werkzeug.
 
 The defaults should be fine if you set up a venv and cellxgene_data folder as above.
 
+## Running cellxgene-gateway with Docker
+
+First, build Docker image:
+
+```bash
+docker build -t cellxgene-gateway .
+```
+
+Then, cellxgene-gateway can be launched as such:
+
+```bash
+docker run -it --rm \
+-v <local_data_dir>:/cellxgene-data \
+-p 5005:5005 \
+cellxgene-gateway
+```
+
+Additional environment variables can be provided with the `-e` parameter:
+
+```bash
+docker run -it --rm \
+-v <local_data_dir>:/cellxgene-data \
+-e GATEWAY_PORT=8080 \
+-p 8080:8080 \
+cellxgene-gateway
+```
+
 # Customization
 
 The current paradigm for customization is to modify files during a build or deployment phase:

SECURITY.md

@@ -0,0 +1,15 @@
+# Security Policy
+
+## Supported Versions
+
+Use this section to tell people about which versions of your project are
+currently being supported with security updates.
+
+| Version | Supported          |
+| ------- | ------------------ |
+| 0.3.2   | :white_check_mark: |
+| <= 0.3.1   | :x:                |
+
+## Reporting a Vulnerability
+
+Please file a bug report issue.

cellxgene_gateway/__init__.py

@@ -7,4 +7,4 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
-__version__ = "0.3.2"
+__version__ = "0.3.10"

cellxgene_gateway/cache_entry.py

@@ -9,7 +9,6 @@
 import datetime
 import logging
 import re
-import urllib.parse
 from enum import Enum
 
 import psutil
@@ -22,6 +21,8 @@ from cellxgene_gateway.cellxgene_exception import CellxgeneException
 from cellxgene_gateway.flask_util import querystring
 from cellxgene_gateway.util import current_time_stamp
 
+logger = logging.getLogger(__name__)
+
 
 class CacheEntryStatus(Enum):
     loaded = "loaded"
@@ -111,7 +112,7 @@ class CacheEntry:
             except psutil.NoSuchProcess:
                 pass
 
-            logging.getLogger("cellxgene_gateway").info(f"terminated {terminated}")
+            logger.info(f"terminated {terminated}")
         self.status = CacheEntryStatus.terminated
 
     def rewrite_text_content(self, cellxgene_content):
@@ -169,38 +170,43 @@ class CacheEntry:
 
         full_path = self.cellxgene_basepath() + subpath + querystring()
 
-        if request.method in ["GET", "HEAD", "OPTIONS"]:
-            cellxgene_response = get(full_path, headers=headers)
-        elif request.method == "PUT":
-            cellxgene_response = put(
-                full_path,
-                headers=headers,
-                data=request.data,
-            )
-        elif request.method == "POST":
-            cellxgene_response = post(
-                full_path,
-                headers=headers,
-                data=request.data,
-            )
-        else:
-            raise CellxgeneException(f"Unexpected method {request.method}", 400)
-        content_type = cellxgene_response.headers["content-type"]
-        if "text" in content_type:
-            gateway_content = self.rewrite_text_content(
-                cellxgene_response.content.decode()
-            )
-        else:
-            gateway_content = cellxgene_response.content
+        try:
+            cellxgene_response = None
+            if request.method in ["GET", "HEAD", "OPTIONS"]:
+                cellxgene_response = get(full_path, headers=headers)
+            elif request.method == "PUT":
+                cellxgene_response = put(
+                    full_path,
+                    headers=headers,
+                    data=request.data,
+                )
+            elif request.method == "POST":
+                cellxgene_response = post(
+                    full_path,
+                    headers=headers,
+                    data=request.data,
+                )
+            else:
+                raise CellxgeneException(f"Unexpected method {request.method}", 400)
+            content_type = cellxgene_response.headers["content-type"]
+            if "text" in content_type:
+                gateway_content = self.rewrite_text_content(
+                    cellxgene_response.content.decode()
+                )
+            else:
+                gateway_content = cellxgene_response.content
 
-        resp_headers = {}
-        for h in copy_headers:
-            if h in cellxgene_response.headers:
-                resp_headers[h] = cellxgene_response.headers[h]
+            resp_headers = {}
+            for h in copy_headers:
+                if h in cellxgene_response.headers:
+                    resp_headers[h] = cellxgene_response.headers[h]
 
-        gateway_response = make_response(
-            gateway_content,
-            cellxgene_response.status_code,
-            resp_headers,
-        )
+            gateway_response = make_response(
+                gateway_content,
+                cellxgene_response.status_code,
+                resp_headers,
+            )
+        finally:
+            if cellxgene_response is not None:
+                cellxgene_response.close()
         return gateway_response

cellxgene_gateway/dir_util.py

@@ -14,44 +14,6 @@ from flask_api import status
 from cellxgene_gateway import env
 from cellxgene_gateway.cellxgene_exception import CellxgeneException
 
-
-def is_subdir(full_path, parent_path):
-    subdir = os.path.realpath(full_path)
-    parent = os.path.realpath(parent_path)
-    return subdir.startswith(parent)
-
-
-def create_dir(parent_path, dir_name):
-    full_path = os.path.join(parent_path, dir_name)
-
-    if "/" in dir_name:
-        raise CellxgeneException(
-            "Please have no slashes in the intended directory.",
-            status.HTTP_400_BAD_REQUEST,
-        )
-    elif not os.path.exists(parent_path):
-        raise CellxgeneException(
-            "The selected User directory does not exist.",
-            status.HTTP_400_BAD_REQUEST,
-        )
-    elif os.path.exists(full_path):
-        raise CellxgeneException(
-            "The provided subdirectory already exists within Directory.",
-            status.HTTP_400_BAD_REQUEST,
-        )
-    elif not is_subdir(full_path, parent_path):
-        raise CellxgeneException(
-            "The directory must be a subdirectory of the parent path.",
-            status.HTTP_400_BAD_REQUEST,
-        )
-    elif not os.path.isdir(parent_path):
-        raise CellxgeneException(
-            "The parent is not a directory.", status.HTTP_400_BAD_REQUEST
-        )
-    else:
-        os.mkdir(full_path)
-
-
 annotations_suffix = "_annotations"
 h5ad_suffix = ".h5ad"
 

cellxgene_gateway/env.py

@@ -9,7 +9,6 @@
 
 import logging
 import os
-import socket
 
 cellxgene_location = os.environ.get("CELLXGENE_LOCATION")
 cellxgene_data = os.environ.get("CELLXGENE_DATA", "")
@@ -24,7 +23,9 @@ external_protocol = os.environ.get(
 )
 ip = os.environ.get("GATEWAY_IP")
 extra_scripts = os.environ.get("GATEWAY_EXTRA_SCRIPTS")
-ttl = os.environ.get("GATEWAY_TTL")
+expire_seconds = int(
+    os.environ.get("GATEWAY_EXPIRE_SECONDS", os.environ.get("GATEWAY_TTL", "3600"))
+)
 enable_annotations = os.environ.get("GATEWAY_ENABLE_ANNOTATIONS", "").lower() in [
     "true",
     "1",
@@ -33,6 +34,7 @@ enable_backed_mode = os.environ.get("GATEWAY_ENABLE_BACKED_MODE", "").lower() in
     "true",
     "1",
 ]
+log_level = logging.getLevelName(os.environ.get("GATEWAY_LOG_LEVEL", "INFO"))
 
 env_vars = {
     "CELLXGENE_LOCATION": cellxgene_location,
@@ -50,9 +52,10 @@ optional_env_vars = {
     "GATEWAY_IP": ip,
     "GATEWAY_PORT": gateway_port,
     "GATEWAY_EXTRA_SCRIPTS": extra_scripts,
-    "GATEWAY_TTL": ttl,
+    "GATEWAY_EXPIRE_SECONDS": expire_seconds,
     "GATEWAY_ENABLE_ANNOTATIONS": enable_annotations,
     "GATEWAY_ENABLE_BACKED_MODE": enable_backed_mode,
+    "GATEWAY_LOG_LEVEL": log_level,
     "CELLXGENE_ARGS": cellxgene_args,
     "CELLXGENE_DATA": cellxgene_data,
     "PROXY_FIX_FOR": proxy_fix_for,

cellxgene_gateway/filecrawl.py

@@ -54,7 +54,7 @@ def render_item_tree(item_tree, item_source):
     if item_tree.descriptor:
         descriptor = item_tree.descriptor.lstrip("/")
         url = f"/filecrawl/{descriptor}?source={item_source.name}"
-        name = descriptor.rsplit("/")[1] if descriptor.find("/") >= 0 else descriptor
+        name = descriptor.rsplit("/", 1)[-1]
         return f"<li><a href='{url}'>{name}</a>{html}</li>"
     else:
         return html

cellxgene_gateway/gateway.py

@@ -22,16 +22,13 @@ from flask import (
     send_from_directory,
     url_for,
 )
-from flask_api import status
 from werkzeug.middleware.proxy_fix import ProxyFix
-from werkzeug.utils import secure_filename
 
 from cellxgene_gateway import env, flask_util
 from cellxgene_gateway.backend_cache import BackendCache
 from cellxgene_gateway.cache_entry import CacheEntryStatus
 from cellxgene_gateway.cache_key import CacheKey
 from cellxgene_gateway.cellxgene_exception import CellxgeneException
-from cellxgene_gateway.dir_util import create_dir, is_subdir
 from cellxgene_gateway.extra_scripts import get_extra_scripts
 from cellxgene_gateway.filecrawl import render_item_source
 from cellxgene_gateway.process_exception import ProcessException
@@ -53,6 +50,14 @@ def _force_https(app):
     return wrapper
 
 
+def set_no_cache(resp):
+    resp.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
+    resp.headers["Pragma"] = "no-cache"
+    resp.headers["Expires"] = "0"
+    resp.headers["Cache-Control"] = "public, max-age=0"
+    return resp
+
+
 app.wsgi_app = _force_https(app.wsgi_app)
 if (
     env.proxy_fix_for > 0
@@ -158,10 +163,7 @@ def filecrawl(path=None):
             path=path,
         )
     )
-    resp.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
-    resp.headers["Pragma"] = "no-cache"
-    resp.headers["Expires"] = "0"
-    resp.headers["Cache-Control"] = "public, max-age=0"
+    set_no_cache(resp)
     return resp
 
 
@@ -210,7 +212,10 @@ def do_view(path, source_name=None):
         match.status == CacheEntryStatus.loaded
         or match.status == CacheEntryStatus.loading
     ):
-        return match.serve_content(path)
+        if source.is_authorized(match.key.descriptor):
+            return match.serve_content(path)
+        else:
+            raise CellxgeneException("User not authorized to access this data", 403)
     elif match.status == CacheEntryStatus.error:
         raise ProcessException.from_cache_entry(match)
 
@@ -268,6 +273,12 @@ def do_terminate(path):
     return redirect(url_for("do_GET_status"), code=302)
 
 
+@app.route("/metadata/ip_address", methods=["GET"])
+def ip_address():
+    resp = make_response(env.ip)
+    return set_no_cache(resp)
+
+
 def launch():
     env.validate()
     if not item_sources or not len(item_sources):
@@ -288,7 +299,7 @@ def launch():
 
 def main():
     logging.basicConfig(
-        level=logging.INFO,
+        level=env.log_level,
         format="%(asctime)s:%(name)s:%(levelname)s:%(message)s",
     )
     cellxgene_data = os.environ.get("CELLXGENE_DATA", None)

cellxgene_gateway/items/file/fileitem_source.py

@@ -51,7 +51,7 @@ class FileItemSource(ItemSource):
         return self.convert_h5ad_path_to_annotation(item.descriptor)
 
     def list_items(self, filter: str = None) -> ItemTree:
-        item_tree = self.scan_directory()
+        item_tree = self.scan_directory("" if filter is None else filter)
 
         """def get_items(dir):
             if dir.branches:
@@ -121,6 +121,9 @@ class FileItemSource(ItemSource):
         if self.is_h5ad_file(full_path):
             return self.shallowitem_from_descriptor(descriptor)
 
+    def is_authorized(self, descriptor):
+        return True
+
     def lookup(self, indescriptor: str) -> LookupResult:
         descriptor = indescriptor.strip("/")
         if descriptor.endswith(self.annotation_file_suffix):

cellxgene_gateway/items/item_source.py

@@ -40,6 +40,10 @@ class ItemSource(ABC):
     def update(self, item: Item) -> None:
         raise Exception('"update" unimplemented')
 
+    @abstractmethod
+    def is_authorized(self, descriptor: str) -> bool:
+        raise Exception('"is_authorized" unimplemented')
+
     @abstractmethod
     def lookup(self, descriptor: str) -> LookupResult:
         raise Exception('"lookup" unimplemented')

cellxgene_gateway/items/s3/s3item_source.py

@@ -7,9 +7,11 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
-from os.path import basename, dirname, join
+import os
+from os.path import basename, dirname
 from typing import List
 
+import flask
 import s3fs
 
 from cellxgene_gateway import dir_util
@@ -18,6 +20,10 @@ from cellxgene_gateway.items.item_source import ItemSource, LookupResult
 from cellxgene_gateway.items.s3.s3item import S3Item
 
 
+def truthy(val: str):
+    return val.lower() in ["true", "1"]
+
+
 class S3ItemSource(ItemSource):
     def __init__(
         self,
@@ -28,7 +34,10 @@ class S3ItemSource(ItemSource):
         annotation_file_suffix=".csv",
     ):
         self._name = name
-        self.s3 = s3fs.S3FileSystem()
+        enable_cache = os.environ.get("S3_ENABLE_LISTINGS_CACHE", "false").lower()
+        assert enable_cache in ["0", "1", "false", "true"]
+        self.use_listings_cache = truthy(enable_cache)
+        self.s3 = s3fs.S3FileSystem(use_listings_cache=self.use_listings_cache)
         if bucket.startswith("s3://"):
             raise Exception(
                 f"Bucket name should not include s3:// prefix, got {bucket}"
@@ -67,6 +76,13 @@ class S3ItemSource(ItemSource):
         item_tree = self.scan_directory("" if filter is None else filter)
         return item_tree
 
+    @property
+    def refresh(self):
+        return (
+            truthy(flask.request.args.get("refresh", default="false"))
+            or not self.use_listings_cache
+        )
+
     def scan_directory(self, directory_key="") -> dict:
         url = self.url(directory_key)
 
@@ -75,7 +91,7 @@ class S3ItemSource(ItemSource):
 
         s3key_map = dict(
             (self.remove_bucket(filepath), "s3://" + filepath)
-            for filepath in sorted(self.s3.ls(url))
+            for filepath in sorted(self.s3.ls(url, refresh=self.refresh))
         )
 
         def is_annotation_dir(dir_s3key):
@@ -113,6 +129,9 @@ class S3ItemSource(ItemSource):
     def update(self, item: S3Item) -> None:
         pass
 
+    def is_authorized(self, descriptor):
+        return True
+
     def lookup_item(self, descriptor):
         full_path = self.url(descriptor)
         if self.is_h5ad_url(full_path):
@@ -163,7 +182,9 @@ class S3ItemSource(ItemSource):
                 self.make_s3item_from_key(
                     basename(annotation), self.remove_bucket(annotation), True
                 )
-                for annotation in sorted(self.s3.ls(annotations_fullpath))
+                for annotation in sorted(
+                    self.s3.ls(annotations_fullpath, refresh=self.refresh)
+                )
                 if annotation.endswith(self.annotation_file_suffix)
                 and self.s3.isfile("s3://" + annotation)
             ]

cellxgene_gateway/path_util.py

@@ -1,41 +0,0 @@
-# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
-# under the Apache License, Version 2.0 (the "License"); you may not use
-# this file except in compliance with the License. You may obtain a copy
-# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
-# required by applicable law or agreed to in writing, software distributed
-# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
-# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
-# the specific language governing permissions and limitations under the License.
-
-import os
-
-from flask_api import status
-
-from cellxgene_gateway.cache_key import CacheKey
-from cellxgene_gateway.cellxgene_exception import CellxgeneException
-from cellxgene_gateway.dir_util import make_h5ad
-
-
-def validate_exists(file_path):
-    if not os.path.exists(file_path):
-        raise CellxgeneException(
-            "File does not exist: " + file_path, status.HTTP_400_BAD_REQUEST
-        )
-
-
-def validate_is_file(file_path):
-    validate_exists(file_path)
-    if not os.path.isfile(file_path):
-        raise CellxgeneException(
-            "Path is not file: " + file_path, status.HTTP_400_BAD_REQUEST
-        )
-    return
-
-
-def validate_is_dir(file_path):
-    validate_exists(file_path)
-    if not os.path.isdir(file_path):
-        raise CellxgeneException(
-            "Path is not dir: " + file_path, status.HTTP_400_BAD_REQUEST
-        )
-    return

cellxgene_gateway/prune_process_cache.py

@@ -18,7 +18,7 @@ logger = logging.getLogger(__name__)
 class PruneProcessCache:
     def __init__(self, cache):
         self.cache = cache
-        self.expire_seconds = 3600 if env.ttl is None else int(env.ttl)
+        self.expire_seconds = env.expire_seconds
 
     def __call__(self):
         while True:
@@ -39,9 +39,9 @@ class PruneProcessCache:
 
         for process in processes_to_delete:
             try:
-                logger.info(f"pruning process {process.pid} ({process.key.dataset})")
+                logger.info(f"pruning process {process.pid} ({process.key.descriptor})")
                 self.cache.prune(process)
             except Exception:
                 logger.exception(
-                    "failed to prune process {process.pid} ({process.dataset})"
+                    "failed to prune process {process.pid} ({process.key.descriptor})"
                 )

cellxgene_gateway/subprocess_backend.py

@@ -17,6 +17,8 @@ from cellxgene_gateway.dir_util import make_annotations
 from cellxgene_gateway.env import cellxgene_args, enable_annotations, enable_backed_mode
 from cellxgene_gateway.process_exception import ProcessException
 
+logger = logging.getLogger(__name__)
+
 
 class SubprocessBackend:
     def __init__(self):
@@ -55,7 +57,7 @@ class SubprocessBackend:
             scripts,
             cache_entry.key.annotation_file_path,
         )
-        logging.getLogger("cellxgene_gateway").info(f"launching {cmd}")
+        logger.info(f"launching {cmd}")
         process = subprocess.Popen(
             [cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True
         )
@@ -84,5 +86,6 @@ class SubprocessBackend:
                 cache_entry.append_output(output)
 
         cache_entry.set_loaded(process.pid)
-
-        return
+        for output in process.communicate():
+            logger.debug(f"cellxgene:{output}")
+        logger.info(f"exiting {cmd}")

cellxgene_gateway/templates/cache_status.html

@@ -75,7 +75,9 @@
                     const el = $(this);
                     const ts = el.text();
                     const dt = new Date(parseInt(ts * 1000));
-                    el.html(`${dt.toISOString()}<br>(${ts})`);
+                    el.prepend(`${dt.toISOString()}<br>(`);
+                    el.append(')');
+
                });
           })
      </script>

environment.yml

@@ -2,13 +2,17 @@ name: cellxgene-gateway
 channels:
 - conda-forge
 dependencies:
-- python=3.7
+- python=3.9
 - requests
 - flask
 - psutil
 - black
+- twine
+- isort
 - coverage
 - pip
 - pip:
+  - pre_commit
   - flask-api
-  - cellxgene>=0.15
+  - werkzeug
+  - cellxgene

requirements.txt

@@ -1,5 +1,6 @@
-cellxgene>=0.15
+cellxgene
 flask
-flask_api
+flask-api
+werkzeug
 psutil
 requests

tests/items/file/test_fileitem_source.py

@@ -1,10 +1,31 @@
 import tempfile
 import unittest
+from unittest.mock import patch
 
 from cellxgene_gateway.items.file.fileitem_source import FileItemSource
 
 
+def stub_join(path):
+    path.join = lambda x, y: x + "/" + y
+
+
 class TestFileItemSource(unittest.TestCase):
+    @patch("os.path")
+    @patch("os.listdir")
+    def test_list_items_GIVEN_no_subpath_THEN_checks_dir(self, listdir, path):
+        stub_join(path)
+        source = FileItemSource("/tmp/unittest", "local")
+        source.list_items()
+        path.exists.assert_called_once_with("/tmp/unittest/")
+
+    @patch("os.path")
+    @patch("os.listdir")
+    def test_list_items_GIVEN_subpath_THEN_checks_subpath(self, listdir, path):
+        stub_join(path)
+        source = FileItemSource("/tmp/unittest", "local")
+        source.list_items("foo")
+        path.exists.assert_called_once_with("/tmp/unittest/foo")
+
     def test_make_fileitem_from_path_GIVEN_annotation_file_THEN_name_lacks_csv(
         self,
     ):

tests/items/s3/test_s3item_source.py

@@ -1,6 +1,7 @@
 import unittest
 from unittest.mock import MagicMock, Mock, patch
 
+from cellxgene_gateway.gateway import app
 from cellxgene_gateway.items.item import ItemType
 from cellxgene_gateway.items.s3.s3item import S3Item
 from cellxgene_gateway.items.s3.s3item_source import S3ItemSource
@@ -38,7 +39,8 @@ class TestScanDirectory(unittest.TestCase):
                     return True
                 raise Exception("exists called with " + path)
 
-            def ls(path):
+            def ls(path, refresh):
+                assert refresh == True
                 if path == "s3://my-bucket/":
                     return [
                         "my-bucket/lvl1",
@@ -80,7 +82,8 @@ class TestScanDirectory(unittest.TestCase):
 
         s3func.return_value = S3Mock
         source = S3ItemSource("my-bucket")
-        tree = source.scan_directory()
+        with app.test_request_context(query_string="refresh=true") as test_context:
+            tree = source.scan_directory()
 
         def s3item_compare(i1, i2, msg=""):
             self.assertEqual(i1.name, i2.name, "name equals")

tests/test_backend_cache.py

@@ -0,0 +1,28 @@
+import unittest
+from unittest.mock import MagicMock, patch
+
+from cellxgene_gateway.backend_cache import is_port_in_use
+
+
+class TestIsPortInUse(unittest.TestCase):
+    @patch("socket.socket")
+    def test_GIVEN_free_port_THEN_returns_true(self, socketMock):
+        connectMock = socketMock()
+        connectMock.connect_ex.return_value = 0
+        connectMock.__enter__.return_value = connectMock
+        self.assertEqual(is_port_in_use(123), True)
+        self.assertTrue(connectMock.__enter__.calledOnce)
+        self.assertTrue(connectMock.__exit__.calledOnce)
+        self.assertTrue(connectMock.connect_ex.calledOnceWith("a"))
+        self.assertTrue(socketMock.calledOnceWith("a"))
+
+    @patch("socket.socket")
+    def test_GIVEN_used_port_THEN_returns_false(self, socketMock):
+        connectMock = socketMock()
+        connectMock.__enter__.return_value = connectMock
+        connectMock.connect_ex.return_value = 1
+        self.assertTrue(connectMock.__enter__.calledOnce)
+        self.assertTrue(connectMock.__exit__.calledOnce)
+        self.assertTrue(connectMock.connect_ex.calledOnceWith("a"))
+        self.assertTrue(socketMock.calledOnceWith("a"))
+        self.assertEqual(is_port_in_use(123), False)

tests/test_dir_util.py

@@ -0,0 +1,35 @@
+import unittest
+from unittest.mock import MagicMock, patch
+
+from cellxgene_gateway.dir_util import ensure_dir_exists, make_annotations, make_h5ad
+
+
+class TestMakeH5ad(unittest.TestCase):
+    def test_GIVEN_annotation_dir_THEN_returns_h5ad(self):
+        self.assertEqual(make_h5ad("pbmc_annotations"), "pbmc.h5ad")
+
+
+class TestMakeAnnotations(unittest.TestCase):
+    def test_GIVEN_h5ad_THEN_returns_annotations(self):
+        self.assertEqual(make_annotations("pbmc.h5ad"), "pbmc_annotations")
+
+
+class TestMakeAnnotations(unittest.TestCase):
+    def test_GIVEN_h5ad_THEN_returns_annotations(self):
+        self.assertEqual(make_annotations("pbmc.h5ad"), "pbmc_annotations")
+
+
+class TestEnsureDirExists(unittest.TestCase):
+    @patch("os.path.exists")
+    @patch("os.makedirs")
+    def test_GIVEN_existing_THEN_does_not_call_makedir(self, makedirsMock, existsMock):
+        existsMock.return_value = True
+        ensure_dir_exists("/foo")
+        makedirsMock.assert_not_called()
+
+    @patch("os.path.exists")
+    @patch("os.makedirs")
+    def test_GIVEN_not_existing_THEN_calls_makedir(self, makedirsMock, existsMock):
+        existsMock.return_value = False
+        ensure_dir_exists("/foo")
+        makedirsMock.assert_called_once_with("/foo")

tests/test_filecrawl.py

@@ -1,7 +1,11 @@
 import unittest
 from unittest.mock import MagicMock, patch
 
-from cellxgene_gateway.filecrawl import render_item, render_item_source
+from cellxgene_gateway.filecrawl import (
+    render_item,
+    render_item_source,
+    render_item_tree,
+)
 from cellxgene_gateway.items.file.fileitem import FileItem
 from cellxgene_gateway.items.file.fileitem_source import FileItemSource
 from cellxgene_gateway.items.item import ItemTree, ItemType
@@ -41,3 +45,15 @@ class TestRenderItemSource(unittest.TestCase):
             rendered,
             "<h6><a href='/filecrawl.html?source=FakeSource'>FakeSource</a>:some_filter</h6><li><a href='/filecrawl/rootdir?source=FakeSource'>rootdir</a><ul></ul></li>",
         )
+
+
+class TestRenderItemTree(unittest.TestCase):
+    @patch("cellxgene_gateway.items.file.fileitem_source.FileItemSource")
+    def test_GIVEN_deep_nested_dirs_THEN_includes_dirs_in_output(self, item_source):
+        item_source.name = "FakeSource"
+        item_tree = ItemTree("foo/bar/baz", [], [])
+        rendered = render_item_tree(item_tree, item_source)
+        self.assertEqual(
+            rendered,
+            "<li><a href='/filecrawl/foo/bar/baz?source=FakeSource'>baz</a><ul></ul></li>",
+        )

tests/test_prune_process_cache.py

@@ -1,13 +1,21 @@
 import unittest
-from unittest.mock import MagicMock, patch
+from unittest.mock import patch, seal
 
 from cellxgene_gateway.backend_cache import BackendCache
-from cellxgene_gateway.cache_entry import CacheEntry
+from cellxgene_gateway.cache_key import CacheKey
+from cellxgene_gateway.items.file.fileitem import FileItem
+from cellxgene_gateway.items.file.fileitem_source import FileItemSource
+from cellxgene_gateway.items.item import ItemType
+
+key = CacheKey(
+    FileItem("/czi/", name="pbmc3k.h5ad", type=ItemType.h5ad),
+    FileItemSource("/tmp", "local"),
+)
 
 
 class TestPruneProcessCache(unittest.TestCase):
     @patch("cellxgene_gateway.util.current_time_stamp", new=lambda: 0)
-    @patch("cellxgene_gateway.env.ttl", new="10")
+    @patch("cellxgene_gateway.env.expire_seconds", new=10)
     @patch("cellxgene_gateway.cache_entry.CacheEntry")
     @patch("cellxgene_gateway.cache_entry.CacheEntry")
     def test_GIVEN_one_old_one_new_THEN_prune_old(self, old, new):
@@ -15,14 +23,23 @@ class TestPruneProcessCache(unittest.TestCase):
 
         cache = BackendCache()
         old.timestamp = -100
+        old.foo = 12
+        old.pid = 1
+        old.key = key
+        old.terminate.return_value = None
+        seal(old)
+        new.key = key
         cache.entry_list.append(old)
         new.timestamp = -5
+        seal(new)
         cache.entry_list.append(new)
         self.assertEqual(len(cache.entry_list), 2)
         ppc = PruneProcessCache(cache)
         ppc.prune()
         self.assertEqual(len(cache.entry_list), 1)
         self.assertEqual(cache.entry_list[0], new)
+        self.assertEqual(cache.entry_list[0], new)
+        self.assertTrue(old.terminate.called)
 
 
 if __name__ == "__main__":