Moved prometheus-flask-exporter into extras_require

.github/workflows/pr-checks.yaml

@@ -6,7 +6,7 @@ on: [push, pull_request]
 
 jobs:
   black:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
     steps:
       - uses: actions/checkout@v2
         name: Checkout repository
@@ -25,7 +25,7 @@ jobs:
           black . --check
   # This job is copied over from `deploy.yaml`
   run-tests:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
     steps:
       - uses: actions/checkout@v2
 
@@ -39,6 +39,7 @@ jobs:
           conda env create -f environment.yml
           eval "$(conda shell.bash hook)"
           conda activate cellxgene-gateway
+          pip install markupsafe==2.0.1 # temporary workaround for jinja2-2.11.3 calling soft_unicode in markupsafe
           python setup.py install
 
       - name: Run tests
@@ -52,39 +53,16 @@ jobs:
           eval "$(conda shell.bash hook)"
           conda activate cellxgene-gateway
           coverage report --fail-under 41
-          coverage report > coverage.txt
-          coverage html -i
           coverage xml -i
 
-      - name: Upload coverage HTML report
+      - name: "Upload coverage to Codecov"
-        uses: actions/upload-artifact@v4
+        uses: codecov/codecov-action@v1
         with:
-          name: coverage-html
+          token: ${{ secrets.CODECOV_TOKEN }}
-          path: htmlcov/
+          files: ./coverage.xml
-          retention-days: 30
+          flags: unittests
-
+          env_vars: OS,PYTHON
-      - name: Upload coverage xml
+          name: codecov-umbrella
-        uses: actions/upload-artifact@v4
+          fail_ci_if_error: true
-        with:
+          path_to_write_report: ./codecov_report.txt
-          name: coverage-xml
+          verbose: true
-          path: coverage.xml
-          retention-days: 30
-
-      - name: Upload coverage summary
-        uses: actions/upload-artifact@v4
-        with:
-          name: coverage-summary
-          path: coverage.txt
-          retention-days: 30
-      # - name: "Upload coverage to Codecov"
-      #   if: ${{ github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) }}
-      #   uses: codecov/codecov-action@v1
-      #   with:
-      #     token: ${{ secrets.CODECOV_TOKEN }}
-      #     files: ./coverage.xml
-      #     flags: unittests
-      #     env_vars: OS,PYTHON
-      #     name: codecov-umbrella
-      #     fail_ci_if_error: true
-      #     path_to_write_report: ./codecov_report.txt
-      #     verbose: true

Changelog.md

@@ -1,22 +1,3 @@
-# 0.4.0
-
-* Removed dependency on flask-api
-* Updated dependencies (python 3.11, numpy, unpinned flask, werkzeug)
-
-# 0.3.12
-
-* #81 List gene set annotations when cell annotations not present
-* #86 Upgrade pip within docker image
-* #73 Moved new link to front
-* #87 Temporarily pin versions of werkzeug and flask
-
-
-# 0.3.11
-
-* #81 added support for gene sets
-* #79 added example for cellxgene-gateway customized docker image
-* #78 prune directories that do not contain h5ad files
-
 # 0.3.10
 
 * #65 Added GATEWAY_EXPIRE_SECONDS to set how long cellxgene servers can remain idle before being terminated.

Dockerfile

@@ -1,7 +1,6 @@
-FROM python:3.11
+FROM python:3.9
 
-RUN pip install --upgrade pip
+RUN pip install cellxgene-gateway 'MarkupSafe<2.1'
-RUN pip install "cellxgene-gateway>=0.4"
 
 ENV CELLXGENE_DATA=/cellxgene-data
 ENV CELLXGENE_LOCATION=/usr/local/bin/cellxgene

README.md

@@ -75,7 +75,7 @@ Optional environment variables:
 * `GATEWAY_PORT` - local port that the gateway should bind to, defaults to 5005
 * `GATEWAY_EXPIRE_SECONDS` - time in seconds that a cellxgene process will remain idle before being terminated. Defaults to 3600 (one hour)
 * `GATEWAY_EXTRA_SCRIPTS` - JSON array of script paths, will be embedded into each page and forwarded with `--scripts` to cellxgene server
-* `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations and gene sets.
+* `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations. 
 * `GATEWAY_ENABLE_BACKED_MODE` - Set to `true` or to `1` to load AnnData in file-backed mode. This saves memory and speeds up launch time but may reduce overall performance.
 * `GATEWAY_LOG_LEVEL` - default is `INFO`. set to `DEBUG` to increase logging and to `WARNING` to decrease logging.
 * `S3_ENABLE_LISTINGS_CACHE` - Set to `true` or to `1` to cache listings of S3 folders for performance. If the cache becomes stale, set `filecrawl.html?refresh=true` query parameter to refresh the cache.
@@ -110,26 +110,11 @@ Additional environment variables can be provided with the `-e` parameter:
 
 ```bash
 docker run -it --rm \
--v ../cellxgene_data:/cellxgene-data \
+-v <local_data_dir>:/cellxgene-data \
 -e GATEWAY_PORT=8080 \
 -p 8080:8080 \
 cellxgene-gateway
 ```
-## Running cellxgene gateway with start scripts
-
-For your convenience, we provide start scripts for flask, gunicorn and uwsgi.
-
-First, set up a .env
-```bash
-cp env_example .env
-# edit .env
-open .env
-```
-
-Then run the scripts in a subshell
-```bash
-( ./start_flask.sh )
-```
 
 # Customization
 
@@ -201,40 +186,26 @@ pip install isort flake8 black
 isort -rc . # rc means recursive, and was deprecated in dev version of isort
 black .
 ```
+## Dependency management
+
+We use the following files for dependency management:
+
+* environment.yml - specifies a conda environment sufficient to run the packages
+* setup.cfg - lists static dependency information, both minimal and extra dependencies
+* setup.py - dynamically generated package information
+* requirements.txt - simple wrapper invoking setup.py
+
+For more details, see the folloiwng links:
+
+* https://towardsdatascience.com/setuptools-python-571e7d5500f2
+* https://towardsdatascience.com/requirements-vs-setuptools-python-ae3ee66e28af
+
+
 
 # Getting Help
 
 If you need help for any reason, please make a github ticket. One of the contributors should help you out.
 
-# Releasing New Versions
-
-## How to prepare for release
-
-- Update Changelog.md and version number in __init__.py
-- Cut a release on github
-	- Go to your project homepage on GitHub
-	- On right side, you will see [Releases](https://github.com/Novartis/cellxgene-gateway/releases) link. Click on it.
-	- Click on Draft a new release
-	- Fill in all the details
-		- Tag version should be the version number of your package release
-		- Release Title can be anything you want, but we use v0.3.11 (the same as the tag to be created on publish)
-		- Description should be changelog
-	- Click Publish release at the bottom of the page
-	- Now under Releases you can view all of your releases.
-	- Copy the download link (tar.gz) and save it somewhere
-
-## How to publish to PyPI
-
-Make sure your `.pypirc` is set up for testpypi and pypi index servers.
- 
-
-```bash
-rm -rf dist
-python setup.py sdist bdist_wheel
-python -m twine upload --repository testpypi dist/*
-python -m twine upload dist/*
-```
-
 # Contributors
 
 * Niket Patel - https://github.com/NiketPatel9

cellxgene_gateway/__init__.py

@@ -7,4 +7,4 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
-__version__ = "0.4.0"
+__version__ = "0.3.10"

cellxgene_gateway/backend_cache.py

@@ -8,10 +8,11 @@
 # the specific language governing permissions and limitations under the License.
 
 import time
-from http import HTTPStatus
 from threading import Thread
 from typing import List
 
+from flask_api import status
+
 from cellxgene_gateway import env
 from cellxgene_gateway.cache_entry import CacheEntry, CacheEntryStatus
 from cellxgene_gateway.cache_key import CacheKey
@@ -52,7 +53,7 @@ class BackendCache:
             return matches[0]
         else:
             raise CellxgeneException(
-                HTTPStatus.INTERNAL_SERVER_ERROR,
+                status.HTTP_500_INTERNAL_SERVER_ERROR,
                 "Found " + str(len(matches)) + " for " + path,
             )
 
@@ -70,7 +71,7 @@ class BackendCache:
             return matches[0]
         else:
             raise CellxgeneException(
-                HTTPStatus.INTERNAL_SERVER_ERROR,
+                status.HTTP_500_INTERNAL_SERVER_ERROR,
                 "Found " + str(len(matches)) + " for " + key.dataset,
             )
 

cellxgene_gateway/cache_entry.py

@@ -58,6 +58,7 @@ class CacheEntry:
 
     @classmethod
     def for_key(cls, key, port):
+
         return cls(
             None,
             key,
@@ -148,7 +149,7 @@ class CacheEntry:
         headers = {}
         copy_headers = [
             "accept",
-            # "accept-encoding" - removed: let requests library handle compression/decompression
+            "accept-encoding",
             "accept-language",
             "cache-control",
             "connection",
@@ -168,8 +169,9 @@ class CacheEntry:
                 headers[h] = request.headers[h]
 
         full_path = self.cellxgene_basepath() + subpath + querystring()
-        cellxgene_response = None
+
         try:
+            cellxgene_response = None
             if request.method in ["GET", "HEAD", "OPTIONS"]:
                 cellxgene_response = get(full_path, headers=headers)
             elif request.method == "PUT":

cellxgene_gateway/dir_util.py

@@ -9,6 +9,8 @@
 
 import os
 
+from flask_api import status
+
 from cellxgene_gateway import env
 from cellxgene_gateway.cellxgene_exception import CellxgeneException
 

cellxgene_gateway/filecrawl.py

@@ -7,32 +7,31 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
-import html
+import os
 import urllib.parse
 
-from cellxgene_gateway import flask_util
+from cellxgene_gateway import env, flask_util
 from cellxgene_gateway.cache_key import CacheKey
 from cellxgene_gateway.dir_util import annotations_suffix, make_annotations, make_h5ad
-from cellxgene_gateway.env import enable_annotations
 
 
 def render_annotations(item, item_source):
-    if not enable_annotations:
-        return ""
     url = flask_util.view_url(
         item_source.get_annotations_subpath(item), item_source.name
     )
-    new_annotation = [f"<a class='new' href='{url}'>new</a>"]
+    new_annotation = f"<a class='new' href='{url}'>new</a>"
-
     annotations = (
+        ", ".join(
             [
-            f"<a href='{CacheKey(item, item_source, a).view_url}/'>{html.escape(a.name)}</a>"
+                f"<a href='{CacheKey(item, item_source, a).view_url}/'>{a.name}</a>"
                 for a in item.annotations
             ]
-        if item.annotations
-        else []
         )
-    return "| annotations: " + ", ".join(new_annotation + annotations)
+        + ", "
+        if item.annotations
+        else ""
+    )
+    return " | annotations: " + annotations + new_annotation
 
 
 def render_item(item, item_source):

cellxgene_gateway/flask_util.py

@@ -26,8 +26,8 @@ def url(endpoint, descriptor, source_name):
 
 
 def view_url(descriptor, source_name):
-    return url("do_view", descriptor, source_name)
+    return url("gateway_blueprint.do_view", descriptor, source_name)
 
 
 def relaunch_url(descriptor, source_name):
-    return url("do_relaunch", descriptor, source_name)
+    return url("gateway_blueprint.do_relaunch", descriptor, source_name)

cellxgene_gateway/gateway.py

@@ -1,51 +1,15 @@
-# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
-# under the Apache License, Version 2.0 (the "License"); you may not use
-# this file except in compliance with the License. You may obtain a copy
-# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
-# required by applicable law or agreed to in writing, software distributed
-# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
-# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
-# the specific language governing permissions and limitations under the License.
-# import BaseHTTPServer
-import json
 import logging
 import os
-import urllib.parse
-from threading import Lock, Thread
 
-from flask import (
+import typer
-    Flask,
+from flask import Flask
-    make_response,
-    redirect,
-    render_template,
-    request,
-    send_from_directory,
-    url_for,
-)
 from werkzeug.middleware.proxy_fix import ProxyFix
 
-from cellxgene_gateway import env, flask_util
+from cellxgene_gateway import env, flask_util, gateway_blueprint
-from cellxgene_gateway.backend_cache import BackendCache
-from cellxgene_gateway.cache_entry import CacheEntryStatus
-from cellxgene_gateway.cache_key import CacheKey
-from cellxgene_gateway.cellxgene_exception import CellxgeneException
-from cellxgene_gateway.extra_scripts import get_extra_scripts
-from cellxgene_gateway.filecrawl import render_item_source
-from cellxgene_gateway.process_exception import ProcessException
-from cellxgene_gateway.prune_process_cache import PruneProcessCache
 from cellxgene_gateway.util import current_time_stamp
 
 app = Flask(__name__)
 
-item_sources = []
-default_item_source = None
-
-# Guard for lazy initialization so tests can import this module without
-# triggering environment-dependent side effects. initialize_data_sources()
-# will set this to True when it has run.
-data_sources_initialized = False
-data_sources_init_lock = Lock()
-
 
 def _force_https(app):
     def wrapper(environ, start_response):
@@ -56,14 +20,6 @@ def _force_https(app):
     return wrapper
 
 
-def set_no_cache(resp):
-    resp.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
-    resp.headers["Pragma"] = "no-cache"
-    resp.headers["Expires"] = "0"
-    resp.headers["Cache-Control"] = "public, max-age=0"
-    return resp
-
-
 app.wsgi_app = _force_https(app.wsgi_app)
 if (
     env.proxy_fix_for > 0
@@ -82,325 +38,44 @@ if (
     )
 
 
-# WSGI middleware to ensure data sources are initialized before the first
+def main(prometheus: bool = False):
-# WSGI request is handled. This guarantees initialization works under
-# Gunicorn/uWSGI (which import the module but don't call main()). The
-# initialize_data_sources() function is idempotent-protected by
-# data_sources_initialized and data_sources_init_lock.
-def _init_on_first_wsgi_request(wsgi_app):
-    def middleware(environ, start_response):
-        global data_sources_initialized
-        if not data_sources_initialized:
-            with data_sources_init_lock:
-                if not app.extensions.get("cellxgene_gateway", {}).get("launchtime"):
-                    app.extensions.setdefault("cellxgene_gateway", {})[
-                        "launchtime"
-                    ] = current_time_stamp()
-
-                if not data_sources_initialized:
-                    initialize_data_sources()
-
-                    env.validate()
-                    if not item_sources or not len(item_sources):
-                        raise Exception(
-                            "No data sources specified for Cellxgene Gateway"
-                        )
-
-                    global default_item_source
-                    if default_item_source is None:
-                        default_item_source = item_sources[0]
-
-                    data_sources_initialized = True
-        return wsgi_app(environ, start_response)
-
-    return middleware
-
-
-# Wrap the WSGI app so Gunicorn/uWSGI will trigger initialization when the
-# first request comes in. Tests that need initialization can call
-# initialize_data_sources() directly.
-app.wsgi_app = _init_on_first_wsgi_request(app.wsgi_app)
-
-cache = BackendCache()
-
-
-# Initialize data sources - this is defined later in the file but called here
-# to ensure initialization happens when WSGI servers (Gunicorn) import the module
-def initialize_data_sources():
-    """Initialize data sources from environment variables.
-    Called at module import time for WSGI server compatibility (Gunicorn).
-    Uses a guard flag to prevent double initialization within a process."""
-    global default_item_source
-
     logging.basicConfig(
         level=env.log_level,
         format="%(asctime)s:%(name)s:%(levelname)s:%(message)s",
     )
-    logger = logging.getLogger(__name__)
-
     cellxgene_data = os.environ.get("CELLXGENE_DATA", None)
     cellxgene_bucket = os.environ.get("CELLXGENE_BUCKET", None)
 
     if cellxgene_bucket is not None:
         from cellxgene_gateway.items.s3.s3item_source import S3ItemSource
 
-        s3_source = S3ItemSource(cellxgene_bucket, name="s3")
+        gateway_blueprint.item_sources.append(S3ItemSource(cellxgene_bucket, name="s3"))
-        item_sources.append(s3_source)
+        default_item_source = "s3"
-        default_item_source = s3_source
-        logger.info("Initialized S3 data source")
-        logger.debug(f"S3 bucket: {cellxgene_bucket}")
     if cellxgene_data is not None:
         from cellxgene_gateway.items.file.fileitem_source import FileItemSource
 
-        file_source = FileItemSource(cellxgene_data, name="local")
+        gateway_blueprint.item_sources.append(
-        item_sources.append(file_source)
+            FileItemSource(cellxgene_data, name="local")
-        default_item_source = file_source
+        )
-        logger.info("Initialized local file data source")
+        default_item_source = "local"
-        logger.debug(f"Data directory: {cellxgene_data}")
+    if len(gateway_blueprint.item_sources) == 0:
-    if len(item_sources) == 0:
         raise Exception("Please specify CELLXGENE_DATA or CELLXGENE_BUCKET")
-    flask_util.include_source_in_url = len(item_sources) > 1
+    flask_util.include_source_in_url = len(gateway_blueprint.item_sources) > 1
 
+    if prometheus:
+        from cellxgene_gateway.prometheus import add_metrics
 
-@app.errorhandler(CellxgeneException)
+        add_metrics(app)
-def handle_invalid_usage(error):
+    app.register_blueprint(gateway_blueprint.gateway_blueprint)
-    message = f"{error.http_status} Error : {error.message}"
+    gateway_blueprint.launch()
 
-    return (
+    app.launchtime = current_time_stamp()
-        render_template(
-            "cellxgene_error.html",
-            extra_scripts=get_extra_scripts(),
-            message=message,
-        ),
-        error.http_status,
-    )
-
-
-@app.errorhandler(ProcessException)
-def handle_invalid_process(error):
-    message = []
-
-    message.append(error.message)
-    message.append(f"{error.http_status} Error.")
-    message.append(f"Stdout: {error.stdout}")
-    message.append(f"Stderr: {error.stderr}")
-
-    return (
-        render_template(
-            "process_error.html",
-            extra_scripts=get_extra_scripts(),
-            message=error.message,
-            http_status=error.http_status,
-            stdout=error.stdout,
-            stderr=error.stderr,
-            relaunch_url=error.key.relaunch_url(),
-            annotation_file=error.key.annotation_descriptor,
-        ),
-        error.http_status,
-    )
-
-
-@app.route("/favicon.ico")
-def favicon():
-    return send_from_directory(
-        os.path.join(app.root_path, "static"),
-        "nibr.ico",
-        mimetype="image/vnd.microsof.icon",
-    )
-
-
-@app.route("/")
-def index():
-    return render_template(
-        "index.html",
-        ip=env.ip,
-        cellxgene_data=env.cellxgene_data,
-        extra_scripts=get_extra_scripts(),
-    )
-
-
-@app.route("/filecrawl.html")
-@app.route("/filecrawl/<path:path>")
-def filecrawl(path=None):
-    source_name = request.args.get("source")
-    sources = (
-        filter(
-            lambda x: x.name == urllib.parse.unquote_plus(source_name),
-            item_sources,
-        )
-        if source_name
-        else item_sources
-    )
-    # loop all data sources --
-    rendered_sources = [
-        render_item_source(item_source, path) for item_source in sources
-    ]  # will we need to make this async in the page???
-    rendered_html = "\n".join(rendered_sources)
-
-    resp = make_response(
-        render_template(
-            "filecrawl.html",
-            extra_scripts=get_extra_scripts(),
-            rendered_html=rendered_html,
-            path=path,
-        )
-    )
-    set_no_cache(resp)
-    return resp
-
-
-entry_lock = Lock()
-
-
-def matching_source(source_name):
-    if source_name is None and default_item_source is not None:
-        source_name = default_item_source.name
-    matching = [i for i in item_sources if i.name == source_name]
-    if len(matching) != 1:
-        raise Exception(f"Could not find matching item source {source_name}")
-    source = matching[0]
-    return source
-
-
-@app.route(
-    "/source/<path:source_name>/view/<path:path>",
-    methods=["GET", "PUT", "POST"],
-)
-@app.route("/view/<path:path>", methods=["GET", "PUT", "POST"])
-def do_view(path, source_name=None):
-    source = matching_source(source_name)
-    match = cache.check_path(source, path)
-
-    if match is None:
-        lookup = source.lookup(path)
-        if lookup is None:
-            raise CellxgeneException(
-                f"Could not find item for path {path} in source {source.name}",
-                404,
-            )
-        key = CacheKey.for_lookup(source, lookup)
-        print(
-            f"view path={path}, source_name={source_name}, dataset={key.file_path}, annotation_file= {key.annotation_file_path}, key={key.descriptor}, source={key.source_name}"
-        )
-        with entry_lock:
-            match = cache.check_entry(key)
-            if match is None:
-                uascripts = get_extra_scripts()
-                match = cache.create_entry(key, uascripts)
-
-    match.timestamp = current_time_stamp()
-
-    if (
-        match.status == CacheEntryStatus.loaded
-        or match.status == CacheEntryStatus.loading
-    ):
-        if source.is_authorized(match.key.descriptor):
-            return match.serve_content(path)
-        else:
-            raise CellxgeneException("User not authorized to access this data", 403)
-    elif match.status == CacheEntryStatus.error:
-        raise ProcessException.from_cache_entry(match)
-    else:
-        raise CellxgeneException(
-            f"Unexpected cache entry status {match.status} for key {match.key.descriptor}",
-            500,
-        )
-
-
-@app.route("/cache_status", methods=["GET"])
-def do_GET_status():
-    return render_template(
-        "cache_status.html",
-        entry_list=cache.entry_list,
-        extra_scripts=get_extra_scripts(),
-    )
-
-
-@app.route("/cache_status.json", methods=["GET"])
-def do_GET_status_json():
-    def map_entry(entry):
-        dataset = entry.key.h5ad_item.descriptor
-        annotation_file = entry.key.annotation_descriptor
-        return {
-            "dataset": dataset,
-            "annotation_file": annotation_file,
-            "launchtime": entry.launchtime,
-            "last_access": entry.timestamp,
-            "status": entry.status.name,
-        }
-
-    return json.dumps(
-        {
-            "launchtime": app.extensions.get("cellxgene_gateway", {}).get("launchtime"),
-            "entry_list": [map_entry(entry) for entry in cache.entry_list],
-        }
-    )
-
-
-def get_cache_key(path):
-    if request.args.get("source_name"):
-        source_name = request.args.get("source_name")
-    elif default_item_source:
-        source_name = default_item_source.name
-    else:
-        source_name = None
-    source = matching_source(source_name)
-    key = CacheKey.for_lookup(source, source.lookup(path))
-    return key
-
-
-@app.route("/relaunch/<path:path>", methods=["GET"])
-def do_relaunch(path):
-    key = get_cache_key(path)
-    match = cache.check_entry(key)
-    if not match is None:
-        match.terminate()
-    return redirect(
-        key.view_url,
-        code=302,
-    )
-
-
-@app.route("/terminate/<path:path>", methods=["GET"])
-def do_terminate(path):
-    key = get_cache_key(path)
-    match = cache.check_entry(key)
-    if not match is None:
-        match.terminate()
-    return redirect(url_for("do_GET_status"), code=302)
-
-
-@app.route("/metadata/ip_address", methods=["GET"])
-def ip_address():
-    resp = make_response(env.ip)
-    return set_no_cache(resp)
-
-
-def start_pruner_thread():
-    pruner = PruneProcessCache(cache)
-    # Run the pruner as a daemon thread so it won't block interpreter
-    # shutdown (for example when Ctrl-C is used in the main thread).
-    # This avoids "Exception ignored in: <module 'threading'...>" at exit.
-    background_thread = Thread(target=pruner, daemon=True)
-    background_thread.start()
-
-
-def launch():
-    start_pruner_thread()
-
-    app.extensions.setdefault("cellxgene_gateway", {})[
-        "launchtime"
-    ] = current_time_stamp()
     app.run(host="0.0.0.0", port=env.gateway_port, debug=False)
 
 
-app.extensions.setdefault("cellxgene_gateway", {})["launchtime"] = None
+def run():
-
+    typer.run(main)
-
-def main():
-    """CLI entry point for Flask development server."""
-    launch()
 
 
 if __name__ == "__main__":
-    main()
+    run()

cellxgene_gateway/gateway_blueprint.py

@@ -0,0 +1,266 @@
+# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
+# under the Apache License, Version 2.0 (the "License"); you may not use
+# this file except in compliance with the License. You may obtain a copy
+# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
+# required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+# the specific language governing permissions and limitations under the License.
+# import BaseHTTPServer
+import json
+import os
+import urllib.parse
+from threading import Lock, Thread
+
+from flask import (
+    Blueprint,
+    current_app,
+    make_response,
+    redirect,
+    render_template,
+    request,
+    send_from_directory,
+    url_for,
+)
+
+from cellxgene_gateway import env
+from cellxgene_gateway.backend_cache import BackendCache
+from cellxgene_gateway.cache_entry import CacheEntryStatus
+from cellxgene_gateway.cache_key import CacheKey
+from cellxgene_gateway.cellxgene_exception import CellxgeneException
+from cellxgene_gateway.extra_scripts import get_extra_scripts
+from cellxgene_gateway.filecrawl import render_item_source
+from cellxgene_gateway.process_exception import ProcessException
+from cellxgene_gateway.prune_process_cache import PruneProcessCache
+from cellxgene_gateway.util import current_time_stamp
+
+gateway_blueprint = Blueprint("gateway_blueprint", __name__)
+item_sources = []
+default_item_source = None
+
+
+def set_no_cache(resp):
+    resp.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
+    resp.headers["Pragma"] = "no-cache"
+    resp.headers["Expires"] = "0"
+    resp.headers["Cache-Control"] = "public, max-age=0"
+    return resp
+
+
+cache = BackendCache()
+
+
+@gateway_blueprint.errorhandler(CellxgeneException)
+def handle_invalid_usage(error):
+
+    message = f"{error.http_status} Error : {error.message}"
+
+    return (
+        render_template(
+            "cellxgene_error.html",
+            extra_scripts=get_extra_scripts(),
+            message=message,
+        ),
+        error.http_status,
+    )
+
+
+@gateway_blueprint.errorhandler(ProcessException)
+def handle_invalid_process(error):
+
+    message = []
+
+    message.append(error.message)
+    message.append(f"{error.http_status} Error.")
+    message.append(f"Stdout: {error.stdout}")
+    message.append(f"Stderr: {error.stderr}")
+
+    return (
+        render_template(
+            "process_error.html",
+            extra_scripts=get_extra_scripts(),
+            message=error.message,
+            http_status=error.http_status,
+            stdout=error.stdout,
+            stderr=error.stderr,
+            relaunch_url=error.key.relaunch_url(),
+            annotation_file=error.key.annotation_descriptor,
+        ),
+        error.http_status,
+    )
+
+
+@gateway_blueprint.route("/favicon.ico")
+def favicon():
+    return send_from_directory(
+        os.path.join(current_app.root_path, "static"),
+        "nibr.ico",
+        mimetype="image/vnd.microsof.icon",
+    )
+
+
+@gateway_blueprint.route("/")
+def index():
+    return render_template(
+        "index.html",
+        ip=env.ip,
+        cellxgene_data=env.cellxgene_data,
+        extra_scripts=get_extra_scripts(),
+    )
+
+
+@gateway_blueprint.route("/filecrawl.html")
+@gateway_blueprint.route("/filecrawl/<path:path>")
+def filecrawl(path=None):
+    source_name = request.args.get("source")
+    sources = (
+        filter(
+            lambda x: x.name == urllib.parse.unquote_plus(source_name),
+            item_sources,
+        )
+        if source_name
+        else item_sources
+    )
+    # loop all data sources --
+    rendered_sources = [
+        render_item_source(item_source, path) for item_source in sources
+    ]  # will we need to make this async in the page???
+    rendered_html = "\n".join(rendered_sources)
+
+    resp = make_response(
+        render_template(
+            "filecrawl.html",
+            extra_scripts=get_extra_scripts(),
+            rendered_html=rendered_html,
+            path=path,
+        )
+    )
+    set_no_cache(resp)
+    return resp
+
+
+entry_lock = Lock()
+
+
+def matching_source(source_name):
+    if source_name is None:
+        source_name = default_item_source.name
+    matching = [i for i in item_sources if i.name == source_name]
+    if len(matching) != 1:
+        raise Exception(f"Could not find matching item source {source_name}")
+    source = matching[0]
+    return source
+
+
+@gateway_blueprint.route(
+    "/source/<path:source_name>/view/<path:path>",
+    methods=["GET", "PUT", "POST"],
+)
+@gateway_blueprint.route("/view/<path:path>", methods=["GET", "PUT", "POST"])
+def do_view(path, source_name=None):
+    source = matching_source(source_name)
+    match = cache.check_path(source, path)
+
+    if match is None:
+        lookup = source.lookup(path)
+        if lookup is None:
+            raise CellxgeneException(
+                f"Could not find item for path {path} in source {source.name}",
+                404,
+            )
+        key = CacheKey.for_lookup(source, lookup)
+        print(
+            f"view path={path}, source_name={source_name}, dataset={key.file_path}, annotation_file= {key.annotation_file_path}, key={key.descriptor}, source={key.source_name}"
+        )
+        with entry_lock:
+            match = cache.check_entry(key)
+            if match is None:
+                uascripts = get_extra_scripts()
+                match = cache.create_entry(key, uascripts)
+
+    match.timestamp = current_time_stamp()
+
+    if (
+        match.status == CacheEntryStatus.loaded
+        or match.status == CacheEntryStatus.loading
+    ):
+        if source.is_authorized(match.key.descriptor):
+            return match.serve_content(path)
+        else:
+            raise CellxgeneException("User not authorized to access this data", 403)
+    elif match.status == CacheEntryStatus.error:
+        raise ProcessException.from_cache_entry(match)
+
+
+@gateway_blueprint.route("/cache_status", methods=["GET"])
+def do_GET_status():
+    return render_template(
+        "cache_status.html",
+        entry_list=cache.entry_list,
+        extra_scripts=get_extra_scripts(),
+    )
+
+
+@gateway_blueprint.route("/cache_status.json", methods=["GET"])
+def do_GET_status_json():
+    return json.dumps(
+        {
+            "launchtime": current_app.launchtime,
+            "entry_list": [
+                {
+                    "dataset": entry.key.dataset,
+                    "annotation_file": entry.key.annotation_file,
+                    "launchtime": entry.launchtime,
+                    "last_access": entry.timestamp,
+                    "status": entry.status,
+                }
+                for entry in cache.entry_list
+            ],
+        }
+    )
+
+
+@gateway_blueprint.route("/relaunch/<path:path>", methods=["GET"])
+def do_relaunch(path):
+    source_name = request.args.get("source_name") or default_item_source.name
+    source = matching_source(source_name)
+    key = CacheKey.for_lookup(source, source.lookup(path))
+    match = cache.check_entry(key)
+    if not match is None:
+        match.terminate()
+    return redirect(
+        key.view_url,
+        code=302,
+    )
+
+
+@gateway_blueprint.route("/terminate/<path:path>", methods=["GET"])
+def do_terminate(path):
+    source_name = request.args.get("source_name") or default_item_source.name
+    source = matching_source(source_name)
+    key = CacheKey.for_lookup(source, source.lookup(path))
+    match = cache.check_entry(key)
+    if not match is None:
+        match.terminate()
+    return redirect(url_for("gateway_blueprint.do_GET_status"), code=302)
+
+
+@gateway_blueprint.route("/metadata/ip_address", methods=["GET"])
+def ip_address():
+    resp = make_response(env.ip)
+    return set_no_cache(resp)
+
+
+def launch():
+    env.validate()
+    if not item_sources or not len(item_sources):
+        raise Exception("No data sources specified for Cellxgene Gateway")
+
+    global default_item_source
+    if default_item_source is None:
+        default_item_source = item_sources[0]
+
+    pruner = PruneProcessCache(cache)
+
+    background_thread = Thread(target=pruner)
+    background_thread.start()

cellxgene_gateway/items/file/fileitem_source.py

@@ -24,22 +24,17 @@ class FileItemSource(ItemSource):
         h5ad_suffix=dir_util.h5ad_suffix,
         annotation_dir_suffix=dir_util.annotations_suffix,
         annotation_file_suffix=".csv",
-        gene_set_file_suffix="_gene_sets.csv",
     ):
         self._name = name
         self.base_path = base_path
         self.h5ad_suffix = h5ad_suffix
         self.annotation_dir_suffix = annotation_dir_suffix
         self.annotation_file_suffix = annotation_file_suffix
-        self.gene_set_file_suffix = gene_set_file_suffix
 
     @property
     def name(self):
         return self._name or f"Files:{self.base_path}"
 
-    def is_gene_set(self, path: str) -> bool:
-        return path.endswith(self.gene_set_file_suffix)
-
     def is_h5ad_file(self, path: str) -> bool:
         return path.endswith(self.h5ad_suffix) and os.path.isfile(path)
 
@@ -68,7 +63,7 @@ class FileItemSource(ItemSource):
 
         return item_tree
 
-    def scan_directory(self, subpath: str = "") -> ItemTree:
+    def scan_directory(self, subpath="") -> dict:
         base_path = os.path.join(self.base_path, subpath)
 
         if not os.path.exists(base_path):
@@ -105,11 +100,6 @@ class FileItemSource(ItemSource):
             branches = [
                 self.scan_directory(os.path.join(subpath, subdir)) for subdir in subdirs
             ]
-            # Exclude branches without files as leaves. Since traversal is applied pre-order,
-            # branch.branches has already been processed and we don't need to check deeper nesting.
-            branches = [
-                branch for branch in branches if branch.items or branch.branches
-            ]
 
         return ItemTree(subpath, items, branches)
 
@@ -186,29 +176,11 @@ class FileItemSource(ItemSource):
         annotations_subpath = self.get_annotations_subpath(item)
         annotations_fullpath = self.full_path(annotations_subpath)
         if os.path.isdir(annotations_fullpath):
-            sorted_files = sorted(os.listdir(annotations_fullpath))
+            return [
-            annotation_files = [
                 self.make_fileitem_from_path(annotation, annotations_subpath, True)
-                for annotation in sorted_files
+                for annotation in sorted(os.listdir(annotations_fullpath))
                 if annotation.endswith(self.annotation_file_suffix)
-                and not self.is_gene_set(annotation)
                 and os.path.isfile(os.path.join(annotations_fullpath, annotation))
             ]
-
-            # Catch gene sets without accompanying [annotations].csv
-            gene_sets_files = [
-                self.make_fileitem_from_path(
-                    annotation[: -len(self.gene_set_file_suffix)] + ".csv",
-                    annotations_subpath,
-                    True,
-                )
-                for annotation in sorted_files
-                if self.is_gene_set(annotation)
-                and annotation[: -len(self.gene_set_file_suffix)]
-                not in [a.name for a in annotation_files]
-                and os.path.isfile(os.path.join(annotations_fullpath, annotation))
-            ]
-
-            return sorted(annotation_files + gene_sets_files, key=lambda x: x.name)
         else:
             return None

cellxgene_gateway/items/s3/s3item_source.py

@@ -116,9 +116,6 @@ class S3ItemSource(ItemSource):
         branches = None
         if len(subdir_keys) > 0:
             branches = [self.scan_directory(key) for key in subdir_keys]
-            branches = [
-                branch for branch in branches if branch.items or branch.branches
-            ]
 
         return ItemTree(directory_key, items, branches)
 

cellxgene_gateway/prometheus.py

@@ -0,0 +1,9 @@
+from prometheus_flask_exporter import PrometheusMetrics
+
+from cellxgene_gateway import __version__
+
+
+def add_metrics(app):
+    metrics = PrometheusMetrics(app)
+    metrics.info("app_info", "Application info", version=__version__)
+    return metrics

cellxgene_gateway/subprocess_backend.py

@@ -9,7 +9,8 @@
 
 import logging
 import subprocess
-from http import HTTPStatus
+
+from flask_api import status
 
 from cellxgene_gateway.cache_entry import CacheEntryStatus
 from cellxgene_gateway.dir_util import make_annotations
@@ -29,11 +30,8 @@ class SubprocessBackend:
                 extra_args = f" --annotations-dir {make_annotations(file_path)}"
             else:
                 extra_args = f" --annotations-file {annotation_file_path}"
-                gene_sets_file_path = annotation_file_path[:-4] + "_gene_sets.csv"
-                extra_args += f" --gene-sets-file {gene_sets_file_path}"
         else:
             extra_args = " --disable-annotations"
-            extra_args += " --disable-gene-sets-save"
         if enable_backed_mode:
             extra_args += " --backed"
         if not cellxgene_args is None:
@@ -75,10 +73,10 @@ class SubprocessBackend:
                     or "Could not open file" in stderr
                 ):
                     message = "File was invalid."
-                    http_status = HTTPStatus.BAD_REQUEST
+                    http_status = status.HTTP_400_BAD_REQUEST
                 else:
                     message = "Cellxgene failed to launch dataset."
-                    http_status = HTTPStatus.INTERNAL_SERVER_ERROR
+                    http_status = status.HTTP_500_INTERNAL_SERVER_ERROR
 
                 cache_entry.status = CacheEntryStatus.error
                 cache_entry.set_error(message, stderr, http_status)

cellxgene_gateway/templates/cache_status.html

@@ -61,7 +61,7 @@
                     <td>
                          {% if entry.status.name == 'loaded' %}
                          <a
-                              href="{{ url_for('do_terminate', path=entry.key.descriptor, source_name=entry.key.source_name) }}">
+                              href="{{ url_for('gateway_blueprint.do_terminate', path=entry.key.descriptor, source_name=entry.key.source_name) }}">
                               terminate </a>
                          {% endif %}
                     </td>

cellxgene_gateway/templates/cellxgene_error.html

@@ -28,11 +28,11 @@
           
      <h4>{{ message }}</h4>
 
-     <a href="{{ url_for('filecrawl') }}">
+     <a href="{{ url_for('gateway_blueprint.filecrawl') }}">
           Please click here to be redirected to the file directory.
      </a>
      <br>
-     <a href="{{ url_for('index') }}">
+     <a href="{{ url_for('gateway_blueprint.index') }}">
           Please click here to return to the homepage.
      </a>
      </div>

cellxgene_gateway/templates/filecrawl.html

@@ -36,10 +36,10 @@
           Navigation:
      <ul>
      {% if path %}
-     <li><a href="{{ url_for('filecrawl') }}">top level</a></li>
+     <li><a href="{{ url_for('gateway_blueprint.filecrawl') }}">top level</a></li>
      {% else %}
      {% endif %}
-     <li><a href="{{ url_for('index') }}">homepage</a></li>
+     <li><a href="{{ url_for('gateway_blueprint.index') }}">homepage</a></li>
      </ul>
      </p>
      <script>

cellxgene_gateway/templates/index.html

@@ -35,12 +35,12 @@
 		Links:	
 	</h1>
 	<div class="list-group" style="width:50%;padding-left:65px">
-	  <a href="{{ url_for('filecrawl') }}" class="list-group-item list-group-item-action">
+	  <a href="{{ url_for('gateway_blueprint.filecrawl') }}" class="list-group-item list-group-item-action">
 		  <u>File Crawler: Allows you to view all uploaded data.</u></a>
 		  
 	</div>
 	<div class="list-group" style="width:50%;padding-left:65px">
-		<a href="{{ url_for('do_GET_status') }}" class="list-group-item list-group-item-action">
+		<a href="{{ url_for('gateway_blueprint.do_GET_status') }}" class="list-group-item list-group-item-action">
 			<u>Cache Status: view status of launched cellxgene servers.</u></a>
 	</div>
 	

cellxgene_gateway/templates/loading.html

@@ -35,11 +35,11 @@
           The page will refresh shortly.
      </p>
 
-     <a href="{{ url_for('filecrawl') }}">
+     <a href="{{ url_for('gateway_blueprint.filecrawl') }}">
           Please click here to be redirected to the file directory.
      </a>
      <br>
-     <a href="{{ url_for('index') }}">
+     <a href="{{ url_for('gateway_blueprint.index') }}">
           Please click here to return to the homepage.
      </a>
      </div>

cellxgene_gateway/templates/process_error.html

@@ -39,10 +39,10 @@
           <li><a href="{{ relaunch_url }}">
                     Attempt to relaunch the cellxgene server.
                </a></li>
-          <li><a href="{{ url_for('filecrawl') }}">
+          <li><a href="{{ url_for('gateway_blueprint.filecrawl') }}">
                Return to the file directory.
           </a></li>
-          <li><a href="{{ url_for('index') }}">
+          <li><a href="{{ url_for('gateway_blueprint.index') }}">
                Return to the homepage.
           </a></li>
      </ul>

env_example

@@ -1,3 +0,0 @@
-export CELLXGENE_LOCATION=$(pwd)/.venv/bin/cellxgene
-export CELLXGENE_DATA=../cellxgene_data
-export GATEWAY_IP=127.0.0.1

environment.yml

@@ -2,16 +2,19 @@ name: cellxgene-gateway
 channels:
 - conda-forge
 dependencies:
-- python=3.11
+- python=3.9
 - requests
 - flask
 - psutil
 - black
+- typer
 - twine
 - isort
 - coverage
 - pip
 - pip:
   - pre_commit
+  - flask-api
   - werkzeug
   - cellxgene
+  - prometheus-flask-exporter

examples/customized_docker_image/Dockerfile

@@ -1,14 +0,0 @@
-FROM python:3.9
-
-RUN pip install cellxgene-gateway 'MarkupSafe<2.1'
-
-COPY customize_ui.sh customize_ui.sh
-RUN CELLXGENE_GATEWAY_DIR=/usr/local/lib/python3.9/site-packages/cellxgene_gateway . ./customize_ui.sh
-
-ENV CELLXGENE_DATA=/cellxgene-data
-ENV CELLXGENE_LOCATION=/usr/local/bin/cellxgene
-EXPOSE 5005
-
-RUN mkdir /cellxgene-data
-
-CMD ["cellxgene-gateway"]

examples/customized_docker_image/README.md

@@ -1,14 +0,0 @@
-# Purpose
-
-This is a simple example of how to make a small script to customize the UI of cellxgene-gateway. The script that does the customization is `customize_ui.sh`, it simply makes the main header green using CSS but you could do anything you want there (including adding more script tags, etc).
-
-# Usage
-
-```
-docker build -t cellxgene_custom .
-CELLXGENE_DATA=`pwd`/../../../cellxgene_data
-docker run -p 5005:5005 --mount src=$CELLXGENE_DATA,target=/cellxgene-data,type=bind cellxgene_custom
-```
-
-If you now open http://localhost:5005 you should see a green cellxgene gateway header.
-

examples/customized_docker_image/customize_ui.sh

@@ -1,3 +0,0 @@
-# make the header bright green
-find "${CELLXGENE_GATEWAY_DIR}/templates" -name index.html -exec sed -i -e 's/<head>/<head>\
-> <style> header h3 {color: #0F0;} <\/style>/g' {} \;

requirements.txt

@@ -1,5 +1,9 @@
-cellxgene
+ # requirements.txt
-flask
+ #
-werkzeug
+ # installs dependencies from ./setup.py, and the package itself,
-psutil
+ # in editable mode
-requests
+ -e .[prometheus]
+
+ # (the -e above is optional). you could also just install the package
+ # normally with just the line below (after uncommenting)
+ # .

run.sh.example

@@ -0,0 +1,6 @@
+export CELLXGENE_LOCATION=$(pwd)/.cellxgene-gateway/bin/cellxgene
+export CELLXGENE_DATA=../cellxgene_data
+export GATEWAY_IP=127.0.0.1
+
+#Once these are set, you run like a normal Flask app
+cellxgene-gateway

setup.cfg

@@ -1,2 +1,30 @@
 [metadata]
-description-file = README.md
+description_file = README.md
+description = "Cellxgene Gateway"
+author = "Niket Patel, Yohann Potier, Alok Saldanha"
+author_email = "alok.saldanha@novartis.com"
+long_description_content_type="text/markdown"
+license = "MIT"
+keywords ="visualization, genomics"
+url = "http://github.com/Novartis/cellxgene-gateway"
+python_requires = ">=3.6"
+classifier =
+    "Topic :: Scientific/Engineering :: Visualization"
+
+[options]
+install_requires =
+    cellxgene
+    flask
+    flask-api
+    werkzeug
+    psutil
+    requests
+    typer
+
+[options.extras_require]
+prometheus =
+    prometheus-flask-exporter
+
+[entry_points]
+console_scripts =
+    cellxgene-gateway = cellxgene_gateway.cli:run

setup.py

@@ -4,9 +4,6 @@ import sys
 
 from setuptools import find_packages, setup
 
-if sys.version_info < (3, 6):
-    sys.exit("Sorry, Python < 3.6 is not supported")
-
 
 def read(rel_path):
     here = os.path.abspath(os.path.dirname(__file__))
@@ -23,33 +20,15 @@ def get_version(rel_path):
         raise RuntimeError("Unable to find version string.")
 
 
-def parse_requirements():
-    reqs = []
-    with open("requirements.txt", "r") as f:
-        for line in f.readlines():
-            reqs.append(line.strip("\n"))
-    return reqs
-
-
 with open("README.md", "r") as fh:
     long_description = fh.read()
 
-install_reqs = parse_requirements()
-
 setup(
-    # mandatory
-    name="cellxgene-gateway",
     # mandatory
     version=get_version("cellxgene_gateway/__init__.py"),
     # mandatory
-    author="Niket Patel, Yohann Potier, Alok Saldanha",
-    author_email="alok.saldanha@novartis.com",
-    description=("Cellxgene Gateway"),
     long_description=long_description,
-    long_description_content_type="text/markdown",
+    name="cellxgene-gateway",
-    license="MIT",
-    keywords="visualization, genomics",
-    url="http://github.com/Novartis/cellxgene-gateway",
     packages=find_packages(),
     package_data={
         "cellxgene_gateway": [
@@ -60,10 +39,4 @@ setup(
         ]
     },
     data_files=[("", ["README.md", "LICENSE"])],
-    install_requires=install_reqs,
-    entry_points={
-        "console_scripts": ["cellxgene-gateway=cellxgene_gateway.gateway:main"]
-    },
-    classifiers=["Topic :: Scientific/Engineering :: Visualization"],
-    python_requires=">=3.6",
 )

start_flask.sh

@@ -1,41 +0,0 @@
-#!/bin/bash
-
-# start_gunicorn.sh - Start Cellxgene Gateway with Gunicorn
-#
-# PREREQUISITES:
-# - Gunicorn installed (included with cellxgene 1.3.0, or: pip install gunicorn)
-# - Virtual environment activated
-# - .env file with CELLXGENE_LOCATION and CELLXGENE_DATA (or CELLXGENE_BUCKET)
-#
-# USAGE:
-# ./start_gunicorn.sh
-
-# Exit on error
-set -e
-
-# Get the directory where this script is located
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-
-# Source environment variables
-echo "Loading environment variables..."
-if [ -f "$SCRIPT_DIR/.env" ]; then
-    source "$SCRIPT_DIR/.env"
-else
-    echo "Error: .env file not found at $SCRIPT_DIR/.env"
-    echo "Please create it with required environment variables"
-    exit 1
-fi
-
-# Verify required environment variables
-if [ -z "$CELLXGENE_LOCATION" ]; then
-    echo "Error: CELLXGENE_LOCATION not set"
-    exit 1
-fi
-
-if [ -z "$CELLXGENE_DATA" ] && [ -z "$CELLXGENE_BUCKET" ]; then
-    echo "Error: Either CELLXGENE_DATA or CELLXGENE_BUCKET must be set"
-    exit 1
-fi
-
-exec cellxgene-gateway

start_gunicorn.sh

@@ -1,91 +0,0 @@
-#!/bin/bash
-
-# start_gunicorn.sh - Start Cellxgene Gateway with Gunicorn
-#
-# PREREQUISITES:
-# - Gunicorn installed (included with cellxgene 1.3.0, or: pip install gunicorn)
-# - Virtual environment activated
-# - .env file with CELLXGENE_LOCATION and CELLXGENE_DATA (or CELLXGENE_BUCKET)
-#
-# USAGE:
-# ./start_gunicorn.sh
-
-# Exit on error
-set -e
-
-# Get the directory where this script is located
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-# Source environment variables
-echo "Loading environment variables..."
-if [ -f "$SCRIPT_DIR/.env" ]; then
-    source "$SCRIPT_DIR/.env"
-else
-    echo "Error: .env file not found at $SCRIPT_DIR/.env"
-    echo "Please create it with required environment variables"
-    exit 1
-fi
-
-# Verify required environment variables
-if [ -z "$CELLXGENE_LOCATION" ]; then
-    echo "Error: CELLXGENE_LOCATION not set"
-    exit 1
-fi
-
-if [ -z "$CELLXGENE_DATA" ] && [ -z "$CELLXGENE_BUCKET" ]; then
-    echo "Error: Either CELLXGENE_DATA or CELLXGENE_BUCKET must be set"
-    exit 1
-fi
-
-# Gunicorn configuration
-# WARNING: Multi-worker mode has cache synchronization issues (see plans/002-shared-cache-implementation.md)
-# Each worker maintains its own in-memory cache, causing 404s for static assets when different
-# workers handle requests for the same dataset. Use GUNICORN_WORKERS=1 until shared cache is implemented.
-WORKERS=${GUNICORN_WORKERS:-1}
-BIND=${GATEWAY_IP:-0.0.0.0}:${GATEWAY_PORT:-5005}
-TIMEOUT=${GUNICORN_TIMEOUT:-120}
-WORKER_CLASS=${GUNICORN_WORKER_CLASS:-sync}
-KEEPALIVE=${GUNICORN_KEEPALIVE:-5}
-LOG_LEVEL=${GUNICORN_LOG_LEVEL:-info}
-
-# Production optimization: enable backed mode to reduce memory usage
-export GATEWAY_ENABLE_BACKED_MODE=${GATEWAY_ENABLE_BACKED_MODE:-true}
-
-# Check if gunicorn is installed
-if ! command -v gunicorn &> /dev/null; then
-    echo "Error: gunicorn not found. Install with: pip install gunicorn"
-    exit 1
-fi
-
-# Display configuration
-echo "Starting Cellxgene Gateway with Gunicorn..."
-echo "Configuration:"
-echo "  Data source: ${CELLXGENE_DATA:-$CELLXGENE_BUCKET}"
-echo "  Binding to: $BIND"
-echo "  Workers: $WORKERS"
-echo "  Worker class: $WORKER_CLASS"
-echo "  Timeout: ${TIMEOUT}s"
-echo "  Keepalive: ${KEEPALIVE}s"
-echo "  Log level: $LOG_LEVEL"
-echo "  Backed mode: ${GATEWAY_ENABLE_BACKED_MODE}"
-echo ""
-
-cd "$SCRIPT_DIR"
-
-# Start Gunicorn with optimized settings
-# Additional options you can add via environment variables:
-# - GUNICORN_MAX_REQUESTS: Restart worker after N requests (prevents memory leaks)
-# - GUNICORN_MAX_REQUESTS_JITTER: Add randomness to max-requests
-exec gunicorn cellxgene_gateway.gateway:app \
-    --workers "$WORKERS" \
-    --worker-class "$WORKER_CLASS" \
-    --bind "$BIND" \
-    --timeout "$TIMEOUT" \
-    --keep-alive "$KEEPALIVE" \
-    --access-logfile - \
-    --error-logfile - \
-    --log-level "$LOG_LEVEL" \
-    --preload \
-    ${GUNICORN_MAX_REQUESTS:+--max-requests "$GUNICORN_MAX_REQUESTS"} \
-    ${GUNICORN_MAX_REQUESTS_JITTER:+--max-requests-jitter "$GUNICORN_MAX_REQUESTS_JITTER"} \
-    "$@"

start_uwsgi.sh

@@ -1,89 +0,0 @@
-#!/bin/bash
-
-# start_uwsgi.sh - Start Cellxgene Gateway with uWSGI
-#
-# PREREQUISITES:
-# - uWSGI installed (pip install uwsgi)
-# - Virtual environment activated
-# - .env file with CELLXGENE_LOCATION and CELLXGENE_DATA (or CELLXGENE_BUCKET)
-#
-# USAGE:
-# ./start_uwsgi.sh
-
-# Exit on error
-set -e
-
-# Get the directory where this script is located
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-# Source environment variables
-echo "Loading environment variables..."
-if [ -f "$SCRIPT_DIR/.env" ]; then
-    source "$SCRIPT_DIR/.env"
-else
-    echo "Error: .env file not found at $SCRIPT_DIR/.env"
-    echo "Please create it with required environment variables"
-    exit 1
-fi
-
-# Verify required environment variables
-if [ -z "$CELLXGENE_LOCATION" ]; then
-    echo "Error: CELLXGENE_LOCATION not set"
-    exit 1
-fi
-
-if [ -z "$CELLXGENE_DATA" ] && [ -z "$CELLXGENE_BUCKET" ]; then
-    echo "Error: Either CELLXGENE_DATA or CELLXGENE_BUCKET must be set"
-    exit 1
-fi
-
-# uWSGI configuration
-# WARNING: Multi-worker mode has cache synchronization issues (see plans/002-shared-cache-implementation.md)
-# Each worker maintains its own in-memory cache, causing 404s for static assets when different
-# workers handle requests for the same dataset. Use UWSGI_WORKERS=1 until shared cache is implemented.
-WORKERS=${UWSGI_WORKERS:-1}
-HOST=${GATEWAY_IP:-0.0.0.0}
-PORT=${GATEWAY_PORT:-5005}
-TIMEOUT=${UWSGI_TIMEOUT:-120}
-THREADS=${UWSGI_THREADS:-1}
-
-# Production optimization: enable backed mode to reduce memory usage
-export GATEWAY_ENABLE_BACKED_MODE=${GATEWAY_ENABLE_BACKED_MODE:-true}
-
-# Check if uwsgi is installed
-if ! command -v uwsgi &> /dev/null; then
-    echo "Error: uwsgi not found. Install with: pip install uwsgi"
-    exit 1
-else
-    # Display configuration
-    echo "Starting Cellxgene Gateway with uWSGI..."
-    echo "Configuration:"
-    echo "  Data source: ${CELLXGENE_DATA:-$CELLXGENE_BUCKET}"
-    echo "  Binding to: $HOST:$PORT"
-    echo "  Workers: $WORKERS"
-    echo "  Threads: $THREADS"
-    echo "  Timeout: ${TIMEOUT}s"
-    echo "  Backed mode: ${GATEWAY_ENABLE_BACKED_MODE}"
-    echo ""
-
-    cd "$SCRIPT_DIR"
-
-    # Start uWSGI with optimized settings
-    # Additional options you can add via environment variables:
-    # - UWSGI_MAX_REQUESTS: Restart worker after N requests (prevents memory leaks)
-    exec uwsgi \
-        --http "$HOST:$PORT" \
-        --module cellxgene_gateway.gateway:app \
-        --workers "$WORKERS" \
-        --threads "$THREADS" \
-        --harakiri "$TIMEOUT" \
-        --master \
-        --enable-threads \
-        --single-interpreter \
-        --need-app \
-        --die-on-term \
-        --log-x-forwarded-for \
-        ${UWSGI_MAX_REQUESTS:+--max-requests "$UWSGI_MAX_REQUESTS"} \
-        "$@"
-fi
-

tests/items/s3/test_s3item_source.py

@@ -1,24 +1,15 @@
 import unittest
-import os
-import shutil
-import tempfile
-
 from unittest.mock import MagicMock, Mock, patch
 
+from flask import Flask
+
+from cellxgene_gateway.gateway_blueprint import gateway_blueprint
 from cellxgene_gateway.items.item import ItemType
 from cellxgene_gateway.items.s3.s3item import S3Item
 from cellxgene_gateway.items.s3.s3item_source import S3ItemSource
-from cellxgene_gateway.gateway import app
 
 
 class TestScanDirectory(unittest.TestCase):
-    def setUp(self):
-
-        self.app = app
-
-    def tearDown(self):
-        pass
-
     @patch("s3fs.S3FileSystem")
     def test_GIVEN_invalid_bucket_THEN_throws_error(self, s3func):
         class S3Mock:
@@ -37,7 +28,6 @@ class TestScanDirectory(unittest.TestCase):
 
     @patch("s3fs.S3FileSystem")
     def test__GIVEN_multilevel_bucket_THEN_properly_recurses_suburls(self, s3func):
-
         class S3Mock:
             def exists(path):
                 if path in [
@@ -94,7 +84,9 @@ class TestScanDirectory(unittest.TestCase):
 
         s3func.return_value = S3Mock
         source = S3ItemSource("my-bucket")
-        with self.app.test_request_context(query_string="refresh=true") as test_context:
+        app = Flask(__name__)
+        app.register_blueprint(gateway_blueprint)
+        with app.test_request_context(query_string="refresh=true") as test_context:
             tree = source.scan_directory()
 
         def s3item_compare(i1, i2, msg=""):

tests/test_cache_entry.py

@@ -1,16 +1,14 @@
 import unittest
-import tempfile
-import os
-import shutil
 
 from flask import Flask
+
 from cellxgene_gateway import flask_util
 from cellxgene_gateway.cache_entry import CacheEntry, CacheEntryStatus
 from cellxgene_gateway.cache_key import CacheKey
-from cellxgene_gateway.items.item import ItemType
+from cellxgene_gateway.gateway_blueprint import gateway_blueprint
 from cellxgene_gateway.items.file.fileitem import FileItem
 from cellxgene_gateway.items.file.fileitem_source import FileItemSource
-from cellxgene_gateway.gateway import app
+from cellxgene_gateway.items.item import ItemType
 
 key = CacheKey(
     FileItem("/czi/", name="pbmc3k.h5ad", type=ItemType.h5ad),
@@ -20,14 +18,12 @@ key = CacheKey(
 
 class TestRenderEntry(unittest.TestCase):
     def setUp(self):
-        self.app = app
+        self.app = Flask(__name__)
+        self.app.register_blueprint(gateway_blueprint)
         self.app_context = self.app.test_request_context()
         self.app_context.push()
         self.client = self.app.test_client()
 
-    def tearDown(self):
-        self.app_context.pop()
-
     def test_GIVEN_key_and_port_THEN_returns_loading_CacheEntry(self):
         entry = CacheEntry.for_key("some-key", 1)
         self.assertEqual(entry.status, CacheEntryStatus.loading)

tests/test_filecrawl.py

@@ -1,9 +1,5 @@
-import os
-import shutil
-import tempfile
 import unittest
-from collections import defaultdict
+from unittest.mock import MagicMock, patch
-from unittest.mock import patch
 
 from cellxgene_gateway.filecrawl import (
     render_item,
@@ -13,101 +9,30 @@ from cellxgene_gateway.filecrawl import (
 from cellxgene_gateway.items.file.fileitem import FileItem
 from cellxgene_gateway.items.file.fileitem_source import FileItemSource
 from cellxgene_gateway.items.item import ItemTree, ItemType
-from cellxgene_gateway.gateway import app
 
 source = FileItemSource("/tmp")
 
 
-def make_entry(subpath="somepath", annotations=None):
-    return FileItem(
-        subpath=subpath,
-        name="entry",
-        ext=".h5ad",
-        type=ItemType.h5ad,
-        annotations=annotations,
-    )
-
-
 class TestRenderEntry(unittest.TestCase):
-    def setUp(self):
-        self.app = app
-        self.app_context = self.app.test_request_context()
-        self.app_context.push()
-
-    def tearDown(self):
-        self.app_context.pop()
-
     def test_GIVEN_path_both_slash_THEN_view_has_single_slash(self):
-        entry = make_entry(subpath="/somepath/")
+        entry = FileItem(subpath="/somepath/", name="entry", type=ItemType.h5ad)
         rendered = render_item(entry, source)
-        self.assertIn("view/somepath/entry.h5ad/'", rendered)
+        self.assertIn("view/somepath/entry/'", rendered)
 
     def test_GIVEN_path_starts_slash_THEN_view_has_single_slash(self):
-        entry = make_entry(subpath="/somepath")
+        entry = FileItem(subpath="/somepath", name="entry", type=ItemType.h5ad)
         rendered = render_item(entry, source)
-        self.assertIn("view/somepath/entry.h5ad/'", rendered)
+        self.assertIn("view/somepath/entry/'", rendered)
 
     def test_GIVEN_path_ends_slash_THEN_view_has_single_slash(self):
-        entry = make_entry(subpath="somepath/")
+        entry = FileItem(subpath="somepath/", name="entry", type=ItemType.h5ad)
         rendered = render_item(entry, source)
-        self.assertIn("view/somepath/entry.h5ad/'", rendered)
+        self.assertIn("view/somepath/entry/'", rendered)
 
     def test_GIVEN_path_no_slash_THEN_view_has_single_slash(self):
-        entry = make_entry(subpath="somepath")
+        entry = FileItem(subpath="somepath", name="entry", type=ItemType.h5ad)
         rendered = render_item(entry, source)
-        self.assertIn("view/somepath/entry.h5ad/'", rendered)
+        self.assertIn("view/somepath/entry/'", rendered)
-
-
-class TestRenderAnnotation(unittest.TestCase):
-
-    def setUp(self):
-        self.app = app
-        self.app_context = self.app.test_request_context()
-        self.app_context.push()
-
-    def tearDown(self):
-        self.app_context.pop()
-
-    @patch("cellxgene_gateway.filecrawl.enable_annotations", new=True)
-    def test_GIVEN_no_annotation_THEN_new_alone(self):
-        entry = make_entry(annotations=None)
-        rendered = render_item(entry, source)
-        self.assertIn(
-            "> | annotations: <a class='new' href='/source/Files:/tmp/view/somepath/entry_annotations'>new</a></li>",
-            rendered,
-        )
-
-    @patch("cellxgene_gateway.filecrawl.enable_annotations", new=True)
-    def test_GIVEN_annotation_THEN_new_before(self):
-        annotation = FileItem(
-            subpath="somepath/entry_annotations",
-            name="annot",
-            ext=".csv",
-            type=ItemType.annotation,
-        )
-        entry = make_entry(annotations=[annotation])
-        rendered = render_item(entry, source)
-        self.assertIn(
-            "> | annotations: <a class='new' href='/source/Files:/tmp/view/somepath/entry_annotations'>new</a>,"
-            " <a href='/source/Files:/tmp/view/somepath/entry_annotations/annot.csv/'>annot</a></li>",
-            rendered,
-        )
-
-    @patch("cellxgene_gateway.filecrawl.enable_annotations", new=True)
-    def test_GIVEN_annotation_THEN_escaped(self):
-        annotation = FileItem(
-            subpath="somepath/entry_annotations",
-            name="hot&cold",
-            ext=".csv",
-            type=ItemType.annotation,
-        )
-        entry = make_entry(annotations=[annotation])
-        rendered = render_item(entry, source)
-        self.assertIn(
-            "> | annotations: <a class='new' href='/source/Files:/tmp/view/somepath/entry_annotations'>new</a>,"
-            " <a href='/source/Files:/tmp/view/somepath/entry_annotations/hot&cold.csv/'>hot&amp;cold</a></li>",
-            rendered,
-        )
 
 
 class TestRenderItemSource(unittest.TestCase):
@@ -123,48 +48,12 @@ class TestRenderItemSource(unittest.TestCase):
 
 
 class TestRenderItemTree(unittest.TestCase):
-    def setUp(self):
-        self.app = app
-        self.app_context = self.app.test_request_context()
-        self.app_context.push()
-
-    def tearDown(self):
-        self.app_context.pop()
-
     @patch("cellxgene_gateway.items.file.fileitem_source.FileItemSource")
     def test_GIVEN_deep_nested_dirs_THEN_includes_dirs_in_output(self, item_source):
         item_source.name = "FakeSource"
-        item_source.get_annotations_subpath = lambda _: "FakeAnnotations"
+        item_tree = ItemTree("foo/bar/baz", [], [])
-        file_item = FileItem(
-            subpath="foo/bar/baz", name="file.h5ad", type=ItemType.h5ad
-        )
-        item_tree = ItemTree("foo/bar/baz", [file_item], [])
         rendered = render_item_tree(item_tree, item_source)
         self.assertEqual(
             rendered,
-            "<li><a href='/filecrawl/foo/bar/baz?source=FakeSource'>baz</a><ul>"
+            "<li><a href='/filecrawl/foo/bar/baz?source=FakeSource'>baz</a><ul></ul></li>",
-            "<li> <a href='/source/FakeSource/view/foo/bar/baz/file.h5ad/'>file.h5ad</a>"
-            " </li></ul></li>",
-        )
-
-    @patch(
-        "os.listdir",
-        side_effect=lambda parent: defaultdict(
-            list, {"tmp": ["foo"], "tmp/foo": ["bar"]}
-        )[parent],
-    )
-    @patch("os.path.exists", return_value=True)
-    def test_GIVEN_dirs_without_h5ad_THEN_excludes_dirs_in_output(
-        self, listdir, exists
-    ):
-        # Directories:
-        # - tmp
-        #   - foo
-        #     - bar (no h5ad files)
-        item_source = FileItemSource("tmp", name="local")
-        item_tree = item_source.list_items("foo")
-        rendered = render_item_tree(item_tree, item_source)
-        self.assertEqual(
-            rendered,
-            "<li><a href='/filecrawl/foo?source=local'>foo</a><ul></ul></li>",
         )

tests/test_gateway_status_json.py

@@ -1,54 +0,0 @@
-import json
-import unittest
-from types import SimpleNamespace
-
-from cellxgene_gateway.gateway import do_GET_status_json, app, cache
-from cellxgene_gateway.cache_entry import CacheEntry, CacheEntryStatus
-
-
-class TestGatewayStatusJson(unittest.TestCase):
-    def test_do_GET_status_json_returns_expected_structure(self):
-        # Create a minimal fake key with required attributes
-        h5ad_item = SimpleNamespace(descriptor="somedir/dataset.h5ad")
-        key = SimpleNamespace(
-            h5ad_item=h5ad_item,
-            annotation_descriptor="somedir/dataset_annotations/foo.csv",
-        )
-
-        # Create a CacheEntry with known launchtime/timestamp/status
-        entry = CacheEntry(
-            None,
-            key,
-            8000,
-            111,
-            222,
-            CacheEntryStatus.loaded,
-            None,
-            None,
-            None,
-            None,
-        )
-
-        # Install into the gateway cache and set app launchtime
-        cache.entry_list = [entry]
-        app.extensions.setdefault("cellxgene_gateway", {})["launchtime"] = "LAUNCH_TIME"
-
-        rv = do_GET_status_json()
-
-        data = json.loads(rv)
-        # top-level launchtime comes from app.extensions
-        self.assertEqual("LAUNCH_TIME", data["launchtime"])
-
-        self.assertIn("entry_list", data)
-        self.assertEqual(1, len(data["entry_list"]))
-
-        e = data["entry_list"][0]
-        self.assertEqual("somedir/dataset.h5ad", e["dataset"])
-        self.assertEqual("somedir/dataset_annotations/foo.csv", e["annotation_file"])
-        self.assertEqual("loaded", e["status"])
-        self.assertEqual(111, e["launchtime"])
-        self.assertEqual(222, e["last_access"])
-
-
-if __name__ == "__main__":
-    unittest.main()

tests/test_subprocess_backend.py

@@ -1,6 +1,7 @@
 import unittest
 from unittest.mock import MagicMock, patch
 
+from cellxgene_gateway.backend_cache import BackendCache
 from cellxgene_gateway.cache_entry import CacheEntry
 from cellxgene_gateway.cache_key import CacheKey
 from cellxgene_gateway.items.file.fileitem import FileItem
@@ -32,46 +33,10 @@ class TestSubprocessBackend(unittest.TestCase):
             backend.launch(cellxgene_loc, scripts, entry)
         popen.assert_called_once_with(
             [
-                "yes | /some/cellxgene launch /tmp/czi/pbmc3k.h5ad --port 8000 --host 127.0.0.1 --disable-annotations --disable-gene-sets-save --scripts http://example.com/script.js --scripts http://example.com/script2.js"
+                "yes | /some/cellxgene launch /tmp/czi/pbmc3k.h5ad --port 8000 --host 127.0.0.1 --disable-annotations --scripts http://example.com/script.js --scripts http://example.com/script2.js"
             ],
             shell=True,
             stderr=-1,
             stdout=-1,
         )
         self.assertEqual("An unexpected error", context.exception.stderr)
-
-    @patch("subprocess.Popen")
-    def test_launch_GIVEN_annotations_enabled_THEN_set_flags(self, popen):
-        subprocess = MagicMock()
-        subprocess.stdout.readline().decode.return_value = (
-            "[cellxgene] Type CTRL-C at any time to exit.\n"
-        )
-        subprocess.stderr.read().decode.return_value = ""
-        popen.return_value = subprocess
-
-        key = CacheKey(
-            FileItem("/czi/", name="pbmc3k.h5ad", type=ItemType.h5ad),
-            FileItemSource("/tmp", "local"),
-            FileItem(
-                "/czi/pbmc3k_annotations/", name="foo.csv", type=ItemType.annotation
-            ),
-        )
-        entry = CacheEntry.for_key(key, 8000)
-        import cellxgene_gateway.subprocess_backend
-
-        cellxgene_gateway.subprocess_backend.enable_annotations = True
-        try:
-            backend = cellxgene_gateway.subprocess_backend.SubprocessBackend()
-            cellxgene_loc = "/some/cellxgene"
-
-            backend.launch(cellxgene_loc, [], entry)
-        finally:
-            cellxgene_gateway.subprocess_backend.enable_annotations = False
-        popen.assert_called_once_with(
-            [
-                "yes | /some/cellxgene launch /tmp/czi/pbmc3k.h5ad --port 8000 --host 127.0.0.1 --annotations-file /tmp/czi/pbmc3k_annotations/foo.csv --gene-sets-file /tmp/czi/pbmc3k_annotations/foo_gene_sets.csv"
-            ],
-            shell=True,
-            stderr=-1,
-            stdout=-1,
-        )