#66 switched from mocks to test request context

#59 add refresh query param to force refresh of S3 cache

.github/workflows/pr-checks.yaml

@@ -39,6 +39,7 @@ jobs:
           conda env create -f environment.yml
           eval "$(conda shell.bash hook)"
           conda activate cellxgene-gateway
+          pip install markupsafe==2.0.1 # temporary workaround for jinja2-2.11.3 calling soft_unicode in markupsafe
           python setup.py install
 
       - name: Run tests

Changelog.md

@@ -1,3 +1,7 @@
+# 0.3.9
+
+* Added S3_ENABLE_LISTINGS_CACHE variable (See README.md)
+
 # 0.3.8
 
 * Fixed bug #57 affecting deeply nested subdirectory listing

Dockerfile

@@ -0,0 +1,8 @@
+FROM python:3.9
+
+RUN pip install cellxgene-gateway 'MarkupSafe<2.1'
+
+ENV CELLXGENE_DATA=/cellxgene-data
+ENV CELLXGENE_LOCATION=/usr/local/bin/cellxgene
+
+CMD ["cellxgene-gateway"]

README.md

@@ -76,6 +76,7 @@ Optional environment variables:
 * `GATEWAY_EXTRA_SCRIPTS` - JSON array of script paths, will be embedded into each page and forwarded with `--scripts` to cellxgene server
 * `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations. 
 * `GATEWAY_ENABLE_BACKED_MODE` - Set to `true` or to `1` to load AnnData in file-backed mode. This saves memory and speeds up launch time but may reduce overall performance.
+* `S3_ENABLE_LISTINGS_CACHE` - Set to `true` or to `1` to cache listings of S3 folders for performance. If the cache becomes stale, set `filecrawl.html?refresh=true` query parameter to refresh the cache.
 
 If any of the following optional variables are set, [ProxyFix](https://werkzeug.palletsprojects.com/en/1.0.x/middleware/proxy_fix/) will be used.
 * `PROXY_FIX_FOR` - Number of upstream proxies setting X-Forwarded-For
@@ -86,6 +87,33 @@ If any of the following optional variables are set, [ProxyFix](https://werkzeug.
 
 The defaults should be fine if you set up a venv and cellxgene_data folder as above.
 
+## Running cellxgene-gateway with Docker
+
+First, build Docker image:
+
+```bash
+docker build -t cellxgene-gateway .
+```
+
+Then, cellxgene-gateway can be launched as such:
+
+```bash
+docker run -it --rm \
+-v <local_data_dir>:/cellxgene-data \
+-p 5005:5005 \
+cellxgene-gateway
+```
+
+Additional environment variables can be provided with the `-e` parameter:
+
+```bash
+docker run -it --rm \
+-v <local_data_dir>:/cellxgene-data \
+-e GATEWAY_PORT=8080 \
+-p 8080:8080 \
+cellxgene-gateway
+```
+
 # Customization
 
 The current paradigm for customization is to modify files during a build or deployment phase:

cellxgene_gateway/__init__.py

@@ -7,4 +7,4 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
-__version__ = "0.3.8"
+__version__ = "0.3.9"

cellxgene_gateway/gateway.py

@@ -22,9 +22,7 @@ from flask import (
     send_from_directory,
     url_for,
 )
-from flask_api import status
 from werkzeug.middleware.proxy_fix import ProxyFix
-from werkzeug.utils import secure_filename
 
 from cellxgene_gateway import env, flask_util
 from cellxgene_gateway.backend_cache import BackendCache

cellxgene_gateway/items/s3/s3item_source.py

@@ -7,9 +7,11 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
-from os.path import basename, dirname, join
+import os
+from os.path import basename, dirname
 from typing import List
 
+import flask
 import s3fs
 
 from cellxgene_gateway import dir_util
@@ -18,6 +20,10 @@ from cellxgene_gateway.items.item_source import ItemSource, LookupResult
 from cellxgene_gateway.items.s3.s3item import S3Item
 
 
+def truthy(val: str):
+    return val.lower() in ["true", "1"]
+
+
 class S3ItemSource(ItemSource):
     def __init__(
         self,
@@ -28,7 +34,10 @@ class S3ItemSource(ItemSource):
         annotation_file_suffix=".csv",
     ):
         self._name = name
-        self.s3 = s3fs.S3FileSystem()
+        enable_cache = os.environ.get("S3_ENABLE_LISTINGS_CACHE", "false").lower()
+        assert enable_cache in ["0", "1", "false", "true"]
+        self.use_listings_cache = truthy(enable_cache)
+        self.s3 = s3fs.S3FileSystem(use_listings_cache=self.use_listings_cache)
         if bucket.startswith("s3://"):
             raise Exception(
                 f"Bucket name should not include s3:// prefix, got {bucket}"
@@ -67,6 +76,13 @@ class S3ItemSource(ItemSource):
         item_tree = self.scan_directory("" if filter is None else filter)
         return item_tree
 
+    @property
+    def refresh(self):
+        return (
+            truthy(flask.request.args.get("refresh", default="false"))
+            or not self.use_listings_cache
+        )
+
     def scan_directory(self, directory_key="") -> dict:
         url = self.url(directory_key)
 
@@ -75,7 +91,7 @@ class S3ItemSource(ItemSource):
 
         s3key_map = dict(
             (self.remove_bucket(filepath), "s3://" + filepath)
-            for filepath in sorted(self.s3.ls(url))
+            for filepath in sorted(self.s3.ls(url, refresh=self.refresh))
         )
 
         def is_annotation_dir(dir_s3key):
@@ -166,7 +182,9 @@ class S3ItemSource(ItemSource):
                 self.make_s3item_from_key(
                     basename(annotation), self.remove_bucket(annotation), True
                 )
-                for annotation in sorted(self.s3.ls(annotations_fullpath))
+                for annotation in sorted(
+                    self.s3.ls(annotations_fullpath, refresh=self.refresh)
+                )
                 if annotation.endswith(self.annotation_file_suffix)
                 and self.s3.isfile("s3://" + annotation)
             ]

environment.yml

@@ -2,9 +2,9 @@ name: cellxgene-gateway
 channels:
 - conda-forge
 dependencies:
-- python=3.7
+- python=3.9
 - requests
-- flask<2.0.0,>=1.0.2
+- flask
 - psutil
 - black
 - twine
@@ -13,6 +13,6 @@ dependencies:
 - pip
 - pip:
   - pre_commit
-  - flask-api==2.0
+  - flask-api
-  - werkzeug==1.0.1
+  - werkzeug
-  - cellxgene>=0.15
+  - cellxgene

requirements.txt

@@ -1,6 +1,6 @@
-cellxgene>=0.15
+cellxgene
-flask<2.0.0,>=1.0.2
+flask
-flask-api==2.0
+flask-api
-werkzeug==1.0.1
+werkzeug
 psutil
 requests

tests/items/s3/test_s3item_source.py

@@ -1,6 +1,7 @@
 import unittest
 from unittest.mock import MagicMock, Mock, patch
 
+from cellxgene_gateway.gateway import app
 from cellxgene_gateway.items.item import ItemType
 from cellxgene_gateway.items.s3.s3item import S3Item
 from cellxgene_gateway.items.s3.s3item_source import S3ItemSource
@@ -38,7 +39,8 @@ class TestScanDirectory(unittest.TestCase):
                     return True
                 raise Exception("exists called with " + path)
 
-            def ls(path):
+            def ls(path, refresh):
+                assert refresh == True
                 if path == "s3://my-bucket/":
                     return [
                         "my-bucket/lvl1",
@@ -80,7 +82,8 @@ class TestScanDirectory(unittest.TestCase):
 
         s3func.return_value = S3Mock
         source = S3ItemSource("my-bucket")
-        tree = source.scan_directory()
+        with app.test_request_context(query_string="refresh=true") as test_context:
+            tree = source.scan_directory()
 
         def s3item_compare(i1, i2, msg=""):
             self.assertEqual(i1.name, i2.name, "name equals")

tests/test_subprocess_backend.py

@@ -33,7 +33,7 @@ class TestSubprocessBackend(unittest.TestCase):
             backend.launch(cellxgene_loc, scripts, entry)
         popen.assert_called_once_with(
             [
-                "yes | /some/cellxgene launch /tmp/czi/pbmc3k.h5ad --port 8000 --host 127.0.0.1 --disable-annotations --disable-gene-sets-save --scripts http://example.com/script.js --scripts http://example.com/script2.js"
+                "yes | /some/cellxgene launch /tmp/czi/pbmc3k.h5ad --port 8000 --host 127.0.0.1 --disable-annotations --scripts http://example.com/script.js --scripts http://example.com/script2.js"
             ],
             shell=True,
             stderr=-1,