Moved prometheus-flask-exporter into extras_require

68 read from subprocess

Changelog.md

@@ -1,3 +1,10 @@
+# 0.3.10
+
+* #65 Added GATEWAY_EXPIRE_SECONDS to set how long cellxgene servers can remain idle before being terminated.
+* Added GATEWAY_LOG_LEVEL to set the log level
+* #68 Close connections after reading response
+* #68 Background thread reads from output of cellxgene process until it exits
+
 # 0.3.9
 
 * Added S3_ENABLE_LISTINGS_CACHE variable (See README.md)

README.md

@@ -73,9 +73,11 @@ Optional environment variables:
 * `EXTERNAL_PROTOCOL` - typically http when running locally, can be https when deployed if the gateway is behind a load balancer or reverse proxy that performs https termination. Default value "http"
 * `GATEWAY_IP` - ip addess of instance gateway is running on, mostly used to display SSH instructions. Defaults to `socket.gethostbyname(socket.gethostname())`
 * `GATEWAY_PORT` - local port that the gateway should bind to, defaults to 5005
+* `GATEWAY_EXPIRE_SECONDS` - time in seconds that a cellxgene process will remain idle before being terminated. Defaults to 3600 (one hour)
 * `GATEWAY_EXTRA_SCRIPTS` - JSON array of script paths, will be embedded into each page and forwarded with `--scripts` to cellxgene server
 * `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations. 
 * `GATEWAY_ENABLE_BACKED_MODE` - Set to `true` or to `1` to load AnnData in file-backed mode. This saves memory and speeds up launch time but may reduce overall performance.
+* `GATEWAY_LOG_LEVEL` - default is `INFO`. set to `DEBUG` to increase logging and to `WARNING` to decrease logging.
 * `S3_ENABLE_LISTINGS_CACHE` - Set to `true` or to `1` to cache listings of S3 folders for performance. If the cache becomes stale, set `filecrawl.html?refresh=true` query parameter to refresh the cache.
 
 If any of the following optional variables are set, [ProxyFix](https://werkzeug.palletsprojects.com/en/1.0.x/middleware/proxy_fix/) will be used.
@@ -184,6 +186,21 @@ pip install isort flake8 black
 isort -rc . # rc means recursive, and was deprecated in dev version of isort
 black .
 ```
+## Dependency management
+
+We use the following files for dependency management:
+
+* environment.yml - specifies a conda environment sufficient to run the packages
+* setup.cfg - lists static dependency information, both minimal and extra dependencies
+* setup.py - dynamically generated package information
+* requirements.txt - simple wrapper invoking setup.py
+
+For more details, see the folloiwng links:
+
+* https://towardsdatascience.com/setuptools-python-571e7d5500f2
+* https://towardsdatascience.com/requirements-vs-setuptools-python-ae3ee66e28af
+
+
 
 # Getting Help
 

cellxgene_gateway/__init__.py

@@ -7,4 +7,4 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
-__version__ = "0.3.9"
+__version__ = "0.3.10"

cellxgene_gateway/cache_entry.py

@@ -9,7 +9,6 @@
 import datetime
 import logging
 import re
-import urllib.parse
 from enum import Enum
 
 import psutil
@@ -22,6 +21,8 @@ from cellxgene_gateway.cellxgene_exception import CellxgeneException
 from cellxgene_gateway.flask_util import querystring
 from cellxgene_gateway.util import current_time_stamp
 
+logger = logging.getLogger(__name__)
+
 
 class CacheEntryStatus(Enum):
     loaded = "loaded"
@@ -111,7 +112,7 @@ class CacheEntry:
             except psutil.NoSuchProcess:
                 pass
 
-            logging.getLogger("cellxgene_gateway").info(f"terminated {terminated}")
+            logger.info(f"terminated {terminated}")
         self.status = CacheEntryStatus.terminated
 
     def rewrite_text_content(self, cellxgene_content):
@@ -169,38 +170,43 @@ class CacheEntry:
 
         full_path = self.cellxgene_basepath() + subpath + querystring()
 
-        if request.method in ["GET", "HEAD", "OPTIONS"]:
-            cellxgene_response = get(full_path, headers=headers)
-        elif request.method == "PUT":
-            cellxgene_response = put(
-                full_path,
-                headers=headers,
-                data=request.data,
-            )
-        elif request.method == "POST":
-            cellxgene_response = post(
-                full_path,
-                headers=headers,
-                data=request.data,
-            )
-        else:
-            raise CellxgeneException(f"Unexpected method {request.method}", 400)
-        content_type = cellxgene_response.headers["content-type"]
-        if "text" in content_type:
-            gateway_content = self.rewrite_text_content(
-                cellxgene_response.content.decode()
-            )
-        else:
-            gateway_content = cellxgene_response.content
+        try:
+            cellxgene_response = None
+            if request.method in ["GET", "HEAD", "OPTIONS"]:
+                cellxgene_response = get(full_path, headers=headers)
+            elif request.method == "PUT":
+                cellxgene_response = put(
+                    full_path,
+                    headers=headers,
+                    data=request.data,
+                )
+            elif request.method == "POST":
+                cellxgene_response = post(
+                    full_path,
+                    headers=headers,
+                    data=request.data,
+                )
+            else:
+                raise CellxgeneException(f"Unexpected method {request.method}", 400)
+            content_type = cellxgene_response.headers["content-type"]
+            if "text" in content_type:
+                gateway_content = self.rewrite_text_content(
+                    cellxgene_response.content.decode()
+                )
+            else:
+                gateway_content = cellxgene_response.content
 
-        resp_headers = {}
-        for h in copy_headers:
-            if h in cellxgene_response.headers:
-                resp_headers[h] = cellxgene_response.headers[h]
+            resp_headers = {}
+            for h in copy_headers:
+                if h in cellxgene_response.headers:
+                    resp_headers[h] = cellxgene_response.headers[h]
 
-        gateway_response = make_response(
-            gateway_content,
-            cellxgene_response.status_code,
-            resp_headers,
-        )
+            gateway_response = make_response(
+                gateway_content,
+                cellxgene_response.status_code,
+                resp_headers,
+            )
+        finally:
+            if cellxgene_response is not None:
+                cellxgene_response.close()
         return gateway_response

cellxgene_gateway/env.py

@@ -9,7 +9,6 @@
 
 import logging
 import os
-import socket
 
 cellxgene_location = os.environ.get("CELLXGENE_LOCATION")
 cellxgene_data = os.environ.get("CELLXGENE_DATA", "")
@@ -24,7 +23,9 @@ external_protocol = os.environ.get(
 )
 ip = os.environ.get("GATEWAY_IP")
 extra_scripts = os.environ.get("GATEWAY_EXTRA_SCRIPTS")
-ttl = os.environ.get("GATEWAY_TTL")
+expire_seconds = int(
+    os.environ.get("GATEWAY_EXPIRE_SECONDS", os.environ.get("GATEWAY_TTL", "3600"))
+)
 enable_annotations = os.environ.get("GATEWAY_ENABLE_ANNOTATIONS", "").lower() in [
     "true",
     "1",
@@ -33,6 +34,7 @@ enable_backed_mode = os.environ.get("GATEWAY_ENABLE_BACKED_MODE", "").lower() in
     "true",
     "1",
 ]
+log_level = logging.getLevelName(os.environ.get("GATEWAY_LOG_LEVEL", "INFO"))
 
 env_vars = {
     "CELLXGENE_LOCATION": cellxgene_location,
@@ -50,9 +52,10 @@ optional_env_vars = {
     "GATEWAY_IP": ip,
     "GATEWAY_PORT": gateway_port,
     "GATEWAY_EXTRA_SCRIPTS": extra_scripts,
-    "GATEWAY_TTL": ttl,
+    "GATEWAY_EXPIRE_SECONDS": expire_seconds,
     "GATEWAY_ENABLE_ANNOTATIONS": enable_annotations,
     "GATEWAY_ENABLE_BACKED_MODE": enable_backed_mode,
+    "GATEWAY_LOG_LEVEL": log_level,
     "CELLXGENE_ARGS": cellxgene_args,
     "CELLXGENE_DATA": cellxgene_data,
     "PROXY_FIX_FOR": proxy_fix_for,

cellxgene_gateway/flask_util.py

@@ -26,8 +26,8 @@ def url(endpoint, descriptor, source_name):
 
 
 def view_url(descriptor, source_name):
-    return url("do_view", descriptor, source_name)
+    return url("gateway_blueprint.do_view", descriptor, source_name)
 
 
 def relaunch_url(descriptor, source_name):
-    return url("do_relaunch", descriptor, source_name)
+    return url("gateway_blueprint.do_relaunch", descriptor, source_name)

cellxgene_gateway/gateway.py

@@ -1,45 +1,15 @@
-# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
-# under the Apache License, Version 2.0 (the "License"); you may not use
-# this file except in compliance with the License. You may obtain a copy
-# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
-# required by applicable law or agreed to in writing, software distributed
-# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
-# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
-# the specific language governing permissions and limitations under the License.
-# import BaseHTTPServer
-import json
 import logging
 import os
-import urllib.parse
-from threading import Lock, Thread
 
-from flask import (
-    Flask,
-    make_response,
-    redirect,
-    render_template,
-    request,
-    send_from_directory,
-    url_for,
-)
+import typer
+from flask import Flask
 from werkzeug.middleware.proxy_fix import ProxyFix
 
-from cellxgene_gateway import env, flask_util
-from cellxgene_gateway.backend_cache import BackendCache
-from cellxgene_gateway.cache_entry import CacheEntryStatus
-from cellxgene_gateway.cache_key import CacheKey
-from cellxgene_gateway.cellxgene_exception import CellxgeneException
-from cellxgene_gateway.extra_scripts import get_extra_scripts
-from cellxgene_gateway.filecrawl import render_item_source
-from cellxgene_gateway.process_exception import ProcessException
-from cellxgene_gateway.prune_process_cache import PruneProcessCache
+from cellxgene_gateway import env, flask_util, gateway_blueprint
 from cellxgene_gateway.util import current_time_stamp
 
 app = Flask(__name__)
 
-item_sources = []
-default_item_source = None
-
 
 def _force_https(app):
     def wrapper(environ, start_response):
@@ -50,14 +20,6 @@ def _force_https(app):
     return wrapper
 
 
-def set_no_cache(resp):
-    resp.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
-    resp.headers["Pragma"] = "no-cache"
-    resp.headers["Expires"] = "0"
-    resp.headers["Cache-Control"] = "public, max-age=0"
-    return resp
-
-
 app.wsgi_app = _force_https(app.wsgi_app)
 if (
     env.proxy_fix_for > 0
@@ -75,231 +37,10 @@ if (
         x_prefix=env.proxy_fix_prefix,
     )
 
-cache = BackendCache()
 
-
-@app.errorhandler(CellxgeneException)
-def handle_invalid_usage(error):
-
-    message = f"{error.http_status} Error : {error.message}"
-
-    return (
-        render_template(
-            "cellxgene_error.html",
-            extra_scripts=get_extra_scripts(),
-            message=message,
-        ),
-        error.http_status,
-    )
-
-
-@app.errorhandler(ProcessException)
-def handle_invalid_process(error):
-
-    message = []
-
-    message.append(error.message)
-    message.append(f"{error.http_status} Error.")
-    message.append(f"Stdout: {error.stdout}")
-    message.append(f"Stderr: {error.stderr}")
-
-    return (
-        render_template(
-            "process_error.html",
-            extra_scripts=get_extra_scripts(),
-            message=error.message,
-            http_status=error.http_status,
-            stdout=error.stdout,
-            stderr=error.stderr,
-            relaunch_url=error.key.relaunch_url(),
-            annotation_file=error.key.annotation_descriptor,
-        ),
-        error.http_status,
-    )
-
-
-@app.route("/favicon.ico")
-def favicon():
-    return send_from_directory(
-        os.path.join(app.root_path, "static"),
-        "nibr.ico",
-        mimetype="image/vnd.microsof.icon",
-    )
-
-
-@app.route("/")
-def index():
-    return render_template(
-        "index.html",
-        ip=env.ip,
-        cellxgene_data=env.cellxgene_data,
-        extra_scripts=get_extra_scripts(),
-    )
-
-
-@app.route("/filecrawl.html")
-@app.route("/filecrawl/<path:path>")
-def filecrawl(path=None):
-    source_name = request.args.get("source")
-    sources = (
-        filter(
-            lambda x: x.name == urllib.parse.unquote_plus(source_name),
-            item_sources,
-        )
-        if source_name
-        else item_sources
-    )
-    # loop all data sources --
-    rendered_sources = [
-        render_item_source(item_source, path) for item_source in sources
-    ]  # will we need to make this async in the page???
-    rendered_html = "\n".join(rendered_sources)
-
-    resp = make_response(
-        render_template(
-            "filecrawl.html",
-            extra_scripts=get_extra_scripts(),
-            rendered_html=rendered_html,
-            path=path,
-        )
-    )
-    set_no_cache(resp)
-    return resp
-
-
-entry_lock = Lock()
-
-
-def matching_source(source_name):
-    if source_name is None:
-        source_name = default_item_source.name
-    matching = [i for i in item_sources if i.name == source_name]
-    if len(matching) != 1:
-        raise Exception(f"Could not find matching item source {source_name}")
-    source = matching[0]
-    return source
-
-
-@app.route(
-    "/source/<path:source_name>/view/<path:path>",
-    methods=["GET", "PUT", "POST"],
-)
-@app.route("/view/<path:path>", methods=["GET", "PUT", "POST"])
-def do_view(path, source_name=None):
-    source = matching_source(source_name)
-    match = cache.check_path(source, path)
-
-    if match is None:
-        lookup = source.lookup(path)
-        if lookup is None:
-            raise CellxgeneException(
-                f"Could not find item for path {path} in source {source.name}",
-                404,
-            )
-        key = CacheKey.for_lookup(source, lookup)
-        print(
-            f"view path={path}, source_name={source_name}, dataset={key.file_path}, annotation_file= {key.annotation_file_path}, key={key.descriptor}, source={key.source_name}"
-        )
-        with entry_lock:
-            match = cache.check_entry(key)
-            if match is None:
-                uascripts = get_extra_scripts()
-                match = cache.create_entry(key, uascripts)
-
-    match.timestamp = current_time_stamp()
-
-    if (
-        match.status == CacheEntryStatus.loaded
-        or match.status == CacheEntryStatus.loading
-    ):
-        if source.is_authorized(match.key.descriptor):
-            return match.serve_content(path)
-        else:
-            raise CellxgeneException("User not authorized to access this data", 403)
-    elif match.status == CacheEntryStatus.error:
-        raise ProcessException.from_cache_entry(match)
-
-
-@app.route("/cache_status", methods=["GET"])
-def do_GET_status():
-    return render_template(
-        "cache_status.html",
-        entry_list=cache.entry_list,
-        extra_scripts=get_extra_scripts(),
-    )
-
-
-@app.route("/cache_status.json", methods=["GET"])
-def do_GET_status_json():
-    return json.dumps(
-        {
-            "launchtime": app.launchtime,
-            "entry_list": [
-                {
-                    "dataset": entry.key.dataset,
-                    "annotation_file": entry.key.annotation_file,
-                    "launchtime": entry.launchtime,
-                    "last_access": entry.timestamp,
-                    "status": entry.status,
-                }
-                for entry in cache.entry_list
-            ],
-        }
-    )
-
-
-@app.route("/relaunch/<path:path>", methods=["GET"])
-def do_relaunch(path):
-    source_name = request.args.get("source_name") or default_item_source.name
-    source = matching_source(source_name)
-    key = CacheKey.for_lookup(source, source.lookup(path))
-    match = cache.check_entry(key)
-    if not match is None:
-        match.terminate()
-    return redirect(
-        key.view_url,
-        code=302,
-    )
-
-
-@app.route("/terminate/<path:path>", methods=["GET"])
-def do_terminate(path):
-    source_name = request.args.get("source_name") or default_item_source.name
-    source = matching_source(source_name)
-    key = CacheKey.for_lookup(source, source.lookup(path))
-    match = cache.check_entry(key)
-    if not match is None:
-        match.terminate()
-    return redirect(url_for("do_GET_status"), code=302)
-
-
-@app.route("/metadata/ip_address", methods=["GET"])
-def ip_address():
-    resp = make_response(env.ip)
-    return set_no_cache(resp)
-
-
-def launch():
-    env.validate()
-    if not item_sources or not len(item_sources):
-        raise Exception("No data sources specified for Cellxgene Gateway")
-
-    global default_item_source
-    if default_item_source is None:
-        default_item_source = item_sources[0]
-
-    pruner = PruneProcessCache(cache)
-
-    background_thread = Thread(target=pruner)
-    background_thread.start()
-
-    app.launchtime = current_time_stamp()
-    app.run(host="0.0.0.0", port=env.gateway_port, debug=False)
-
-
-def main():
+def main(prometheus: bool = False):
     logging.basicConfig(
-        level=logging.INFO,
+        level=env.log_level,
         format="%(asctime)s:%(name)s:%(levelname)s:%(message)s",
     )
     cellxgene_data = os.environ.get("CELLXGENE_DATA", None)
@@ -308,19 +49,33 @@ def main():
     if cellxgene_bucket is not None:
         from cellxgene_gateway.items.s3.s3item_source import S3ItemSource
 
-        item_sources.append(S3ItemSource(cellxgene_bucket, name="s3"))
+        gateway_blueprint.item_sources.append(S3ItemSource(cellxgene_bucket, name="s3"))
         default_item_source = "s3"
     if cellxgene_data is not None:
         from cellxgene_gateway.items.file.fileitem_source import FileItemSource
 
-        item_sources.append(FileItemSource(cellxgene_data, name="local"))
+        gateway_blueprint.item_sources.append(
+            FileItemSource(cellxgene_data, name="local")
+        )
         default_item_source = "local"
-    if len(item_sources) == 0:
+    if len(gateway_blueprint.item_sources) == 0:
         raise Exception("Please specify CELLXGENE_DATA or CELLXGENE_BUCKET")
-    flask_util.include_source_in_url = len(item_sources) > 1
+    flask_util.include_source_in_url = len(gateway_blueprint.item_sources) > 1
 
-    launch()
+    if prometheus:
+        from cellxgene_gateway.prometheus import add_metrics
+
+        add_metrics(app)
+    app.register_blueprint(gateway_blueprint.gateway_blueprint)
+    gateway_blueprint.launch()
+
+    app.launchtime = current_time_stamp()
+    app.run(host="0.0.0.0", port=env.gateway_port, debug=False)
+
+
+def run():
+    typer.run(main)
 
 
 if __name__ == "__main__":
-    main()
+    run()

cellxgene_gateway/gateway_blueprint.py

@@ -0,0 +1,266 @@
+# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
+# under the Apache License, Version 2.0 (the "License"); you may not use
+# this file except in compliance with the License. You may obtain a copy
+# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
+# required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+# the specific language governing permissions and limitations under the License.
+# import BaseHTTPServer
+import json
+import os
+import urllib.parse
+from threading import Lock, Thread
+
+from flask import (
+    Blueprint,
+    current_app,
+    make_response,
+    redirect,
+    render_template,
+    request,
+    send_from_directory,
+    url_for,
+)
+
+from cellxgene_gateway import env
+from cellxgene_gateway.backend_cache import BackendCache
+from cellxgene_gateway.cache_entry import CacheEntryStatus
+from cellxgene_gateway.cache_key import CacheKey
+from cellxgene_gateway.cellxgene_exception import CellxgeneException
+from cellxgene_gateway.extra_scripts import get_extra_scripts
+from cellxgene_gateway.filecrawl import render_item_source
+from cellxgene_gateway.process_exception import ProcessException
+from cellxgene_gateway.prune_process_cache import PruneProcessCache
+from cellxgene_gateway.util import current_time_stamp
+
+gateway_blueprint = Blueprint("gateway_blueprint", __name__)
+item_sources = []
+default_item_source = None
+
+
+def set_no_cache(resp):
+    resp.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
+    resp.headers["Pragma"] = "no-cache"
+    resp.headers["Expires"] = "0"
+    resp.headers["Cache-Control"] = "public, max-age=0"
+    return resp
+
+
+cache = BackendCache()
+
+
+@gateway_blueprint.errorhandler(CellxgeneException)
+def handle_invalid_usage(error):
+
+    message = f"{error.http_status} Error : {error.message}"
+
+    return (
+        render_template(
+            "cellxgene_error.html",
+            extra_scripts=get_extra_scripts(),
+            message=message,
+        ),
+        error.http_status,
+    )
+
+
+@gateway_blueprint.errorhandler(ProcessException)
+def handle_invalid_process(error):
+
+    message = []
+
+    message.append(error.message)
+    message.append(f"{error.http_status} Error.")
+    message.append(f"Stdout: {error.stdout}")
+    message.append(f"Stderr: {error.stderr}")
+
+    return (
+        render_template(
+            "process_error.html",
+            extra_scripts=get_extra_scripts(),
+            message=error.message,
+            http_status=error.http_status,
+            stdout=error.stdout,
+            stderr=error.stderr,
+            relaunch_url=error.key.relaunch_url(),
+            annotation_file=error.key.annotation_descriptor,
+        ),
+        error.http_status,
+    )
+
+
+@gateway_blueprint.route("/favicon.ico")
+def favicon():
+    return send_from_directory(
+        os.path.join(current_app.root_path, "static"),
+        "nibr.ico",
+        mimetype="image/vnd.microsof.icon",
+    )
+
+
+@gateway_blueprint.route("/")
+def index():
+    return render_template(
+        "index.html",
+        ip=env.ip,
+        cellxgene_data=env.cellxgene_data,
+        extra_scripts=get_extra_scripts(),
+    )
+
+
+@gateway_blueprint.route("/filecrawl.html")
+@gateway_blueprint.route("/filecrawl/<path:path>")
+def filecrawl(path=None):
+    source_name = request.args.get("source")
+    sources = (
+        filter(
+            lambda x: x.name == urllib.parse.unquote_plus(source_name),
+            item_sources,
+        )
+        if source_name
+        else item_sources
+    )
+    # loop all data sources --
+    rendered_sources = [
+        render_item_source(item_source, path) for item_source in sources
+    ]  # will we need to make this async in the page???
+    rendered_html = "\n".join(rendered_sources)
+
+    resp = make_response(
+        render_template(
+            "filecrawl.html",
+            extra_scripts=get_extra_scripts(),
+            rendered_html=rendered_html,
+            path=path,
+        )
+    )
+    set_no_cache(resp)
+    return resp
+
+
+entry_lock = Lock()
+
+
+def matching_source(source_name):
+    if source_name is None:
+        source_name = default_item_source.name
+    matching = [i for i in item_sources if i.name == source_name]
+    if len(matching) != 1:
+        raise Exception(f"Could not find matching item source {source_name}")
+    source = matching[0]
+    return source
+
+
+@gateway_blueprint.route(
+    "/source/<path:source_name>/view/<path:path>",
+    methods=["GET", "PUT", "POST"],
+)
+@gateway_blueprint.route("/view/<path:path>", methods=["GET", "PUT", "POST"])
+def do_view(path, source_name=None):
+    source = matching_source(source_name)
+    match = cache.check_path(source, path)
+
+    if match is None:
+        lookup = source.lookup(path)
+        if lookup is None:
+            raise CellxgeneException(
+                f"Could not find item for path {path} in source {source.name}",
+                404,
+            )
+        key = CacheKey.for_lookup(source, lookup)
+        print(
+            f"view path={path}, source_name={source_name}, dataset={key.file_path}, annotation_file= {key.annotation_file_path}, key={key.descriptor}, source={key.source_name}"
+        )
+        with entry_lock:
+            match = cache.check_entry(key)
+            if match is None:
+                uascripts = get_extra_scripts()
+                match = cache.create_entry(key, uascripts)
+
+    match.timestamp = current_time_stamp()
+
+    if (
+        match.status == CacheEntryStatus.loaded
+        or match.status == CacheEntryStatus.loading
+    ):
+        if source.is_authorized(match.key.descriptor):
+            return match.serve_content(path)
+        else:
+            raise CellxgeneException("User not authorized to access this data", 403)
+    elif match.status == CacheEntryStatus.error:
+        raise ProcessException.from_cache_entry(match)
+
+
+@gateway_blueprint.route("/cache_status", methods=["GET"])
+def do_GET_status():
+    return render_template(
+        "cache_status.html",
+        entry_list=cache.entry_list,
+        extra_scripts=get_extra_scripts(),
+    )
+
+
+@gateway_blueprint.route("/cache_status.json", methods=["GET"])
+def do_GET_status_json():
+    return json.dumps(
+        {
+            "launchtime": current_app.launchtime,
+            "entry_list": [
+                {
+                    "dataset": entry.key.dataset,
+                    "annotation_file": entry.key.annotation_file,
+                    "launchtime": entry.launchtime,
+                    "last_access": entry.timestamp,
+                    "status": entry.status,
+                }
+                for entry in cache.entry_list
+            ],
+        }
+    )
+
+
+@gateway_blueprint.route("/relaunch/<path:path>", methods=["GET"])
+def do_relaunch(path):
+    source_name = request.args.get("source_name") or default_item_source.name
+    source = matching_source(source_name)
+    key = CacheKey.for_lookup(source, source.lookup(path))
+    match = cache.check_entry(key)
+    if not match is None:
+        match.terminate()
+    return redirect(
+        key.view_url,
+        code=302,
+    )
+
+
+@gateway_blueprint.route("/terminate/<path:path>", methods=["GET"])
+def do_terminate(path):
+    source_name = request.args.get("source_name") or default_item_source.name
+    source = matching_source(source_name)
+    key = CacheKey.for_lookup(source, source.lookup(path))
+    match = cache.check_entry(key)
+    if not match is None:
+        match.terminate()
+    return redirect(url_for("gateway_blueprint.do_GET_status"), code=302)
+
+
+@gateway_blueprint.route("/metadata/ip_address", methods=["GET"])
+def ip_address():
+    resp = make_response(env.ip)
+    return set_no_cache(resp)
+
+
+def launch():
+    env.validate()
+    if not item_sources or not len(item_sources):
+        raise Exception("No data sources specified for Cellxgene Gateway")
+
+    global default_item_source
+    if default_item_source is None:
+        default_item_source = item_sources[0]
+
+    pruner = PruneProcessCache(cache)
+
+    background_thread = Thread(target=pruner)
+    background_thread.start()

cellxgene_gateway/prometheus.py

@@ -0,0 +1,9 @@
+from prometheus_flask_exporter import PrometheusMetrics
+
+from cellxgene_gateway import __version__
+
+
+def add_metrics(app):
+    metrics = PrometheusMetrics(app)
+    metrics.info("app_info", "Application info", version=__version__)
+    return metrics

cellxgene_gateway/prune_process_cache.py

@@ -18,7 +18,7 @@ logger = logging.getLogger(__name__)
 class PruneProcessCache:
     def __init__(self, cache):
         self.cache = cache
-        self.expire_seconds = 3600 if env.ttl is None else int(env.ttl)
+        self.expire_seconds = env.expire_seconds
 
     def __call__(self):
         while True:

cellxgene_gateway/subprocess_backend.py

@@ -17,6 +17,8 @@ from cellxgene_gateway.dir_util import make_annotations
 from cellxgene_gateway.env import cellxgene_args, enable_annotations, enable_backed_mode
 from cellxgene_gateway.process_exception import ProcessException
 
+logger = logging.getLogger(__name__)
+
 
 class SubprocessBackend:
     def __init__(self):
@@ -55,7 +57,7 @@ class SubprocessBackend:
             scripts,
             cache_entry.key.annotation_file_path,
         )
-        logging.getLogger("cellxgene_gateway").info(f"launching {cmd}")
+        logger.info(f"launching {cmd}")
         process = subprocess.Popen(
             [cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True
         )
@@ -84,5 +86,6 @@ class SubprocessBackend:
                 cache_entry.append_output(output)
 
         cache_entry.set_loaded(process.pid)
-
-        return
+        for output in process.communicate():
+            logger.debug(f"cellxgene:{output}")
+        logger.info(f"exiting {cmd}")

cellxgene_gateway/templates/cache_status.html

@@ -61,7 +61,7 @@
                     <td>
                          {% if entry.status.name == 'loaded' %}
                          <a
-                              href="{{ url_for('do_terminate', path=entry.key.descriptor, source_name=entry.key.source_name) }}">
+                              href="{{ url_for('gateway_blueprint.do_terminate', path=entry.key.descriptor, source_name=entry.key.source_name) }}">
                               terminate </a>
                          {% endif %}
                     </td>

cellxgene_gateway/templates/cellxgene_error.html

@@ -28,11 +28,11 @@
           
      <h4>{{ message }}</h4>
 
-     <a href="{{ url_for('filecrawl') }}">
+     <a href="{{ url_for('gateway_blueprint.filecrawl') }}">
           Please click here to be redirected to the file directory.
      </a>
      <br>
-     <a href="{{ url_for('index') }}">
+     <a href="{{ url_for('gateway_blueprint.index') }}">
           Please click here to return to the homepage.
      </a>
      </div>

cellxgene_gateway/templates/filecrawl.html

@@ -36,10 +36,10 @@
           Navigation:
      <ul>
      {% if path %}
-     <li><a href="{{ url_for('filecrawl') }}">top level</a></li>
+     <li><a href="{{ url_for('gateway_blueprint.filecrawl') }}">top level</a></li>
      {% else %}
      {% endif %}
-     <li><a href="{{ url_for('index') }}">homepage</a></li>
+     <li><a href="{{ url_for('gateway_blueprint.index') }}">homepage</a></li>
      </ul>
      </p>
      <script>

cellxgene_gateway/templates/index.html

@@ -35,12 +35,12 @@
 		Links:	
 	</h1>
 	<div class="list-group" style="width:50%;padding-left:65px">
-	  <a href="{{ url_for('filecrawl') }}" class="list-group-item list-group-item-action">
+	  <a href="{{ url_for('gateway_blueprint.filecrawl') }}" class="list-group-item list-group-item-action">
 		  <u>File Crawler: Allows you to view all uploaded data.</u></a>
 		  
 	</div>
 	<div class="list-group" style="width:50%;padding-left:65px">
-		<a href="{{ url_for('do_GET_status') }}" class="list-group-item list-group-item-action">
+		<a href="{{ url_for('gateway_blueprint.do_GET_status') }}" class="list-group-item list-group-item-action">
 			<u>Cache Status: view status of launched cellxgene servers.</u></a>
 	</div>
 	

cellxgene_gateway/templates/loading.html

@@ -35,11 +35,11 @@
           The page will refresh shortly.
      </p>
 
-     <a href="{{ url_for('filecrawl') }}">
+     <a href="{{ url_for('gateway_blueprint.filecrawl') }}">
           Please click here to be redirected to the file directory.
      </a>
      <br>
-     <a href="{{ url_for('index') }}">
+     <a href="{{ url_for('gateway_blueprint.index') }}">
           Please click here to return to the homepage.
      </a>
      </div>

cellxgene_gateway/templates/process_error.html

@@ -39,10 +39,10 @@
           <li><a href="{{ relaunch_url }}">
                     Attempt to relaunch the cellxgene server.
                </a></li>
-          <li><a href="{{ url_for('filecrawl') }}">
+          <li><a href="{{ url_for('gateway_blueprint.filecrawl') }}">
                Return to the file directory.
           </a></li>
-          <li><a href="{{ url_for('index') }}">
+          <li><a href="{{ url_for('gateway_blueprint.index') }}">
                Return to the homepage.
           </a></li>
      </ul>

environment.yml

@@ -7,6 +7,7 @@ dependencies:
 - flask
 - psutil
 - black
+- typer
 - twine
 - isort
 - coverage
@@ -16,3 +17,4 @@ dependencies:
   - flask-api
   - werkzeug
   - cellxgene
+  - prometheus-flask-exporter

requirements.txt

@@ -1,6 +1,9 @@
-cellxgene
-flask
-flask-api
-werkzeug
-psutil
-requests
+ # requirements.txt
+ #
+ # installs dependencies from ./setup.py, and the package itself,
+ # in editable mode
+ -e .[prometheus]
+
+ # (the -e above is optional). you could also just install the package
+ # normally with just the line below (after uncommenting)
+ # .

setup.cfg

@@ -1,2 +1,30 @@
 [metadata]
-description-file = README.md
+description_file = README.md
+description = "Cellxgene Gateway"
+author = "Niket Patel, Yohann Potier, Alok Saldanha"
+author_email = "alok.saldanha@novartis.com"
+long_description_content_type="text/markdown"
+license = "MIT"
+keywords ="visualization, genomics"
+url = "http://github.com/Novartis/cellxgene-gateway"
+python_requires = ">=3.6"
+classifier =
+    "Topic :: Scientific/Engineering :: Visualization"
+
+[options]
+install_requires =
+    cellxgene
+    flask
+    flask-api
+    werkzeug
+    psutil
+    requests
+    typer
+
+[options.extras_require]
+prometheus =
+    prometheus-flask-exporter
+
+[entry_points]
+console_scripts =
+    cellxgene-gateway = cellxgene_gateway.cli:run

setup.py

@@ -4,9 +4,6 @@ import sys
 
 from setuptools import find_packages, setup
 
-if sys.version_info < (3, 6):
-    sys.exit("Sorry, Python < 3.6 is not supported")
-
 
 def read(rel_path):
     here = os.path.abspath(os.path.dirname(__file__))
@@ -23,33 +20,15 @@ def get_version(rel_path):
         raise RuntimeError("Unable to find version string.")
 
 
-def parse_requirements():
-    reqs = []
-    with open("requirements.txt", "r") as f:
-        for line in f.readlines():
-            reqs.append(line.strip("\n"))
-    return reqs
-
-
 with open("README.md", "r") as fh:
     long_description = fh.read()
 
-install_reqs = parse_requirements()
-
 setup(
-    # mandatory
-    name="cellxgene-gateway",
     # mandatory
     version=get_version("cellxgene_gateway/__init__.py"),
     # mandatory
-    author="Niket Patel, Yohann Potier, Alok Saldanha",
-    author_email="alok.saldanha@novartis.com",
-    description=("Cellxgene Gateway"),
     long_description=long_description,
-    long_description_content_type="text/markdown",
-    license="MIT",
-    keywords="visualization, genomics",
-    url="http://github.com/Novartis/cellxgene-gateway",
+    name="cellxgene-gateway",
     packages=find_packages(),
     package_data={
         "cellxgene_gateway": [
@@ -60,10 +39,4 @@ setup(
         ]
     },
     data_files=[("", ["README.md", "LICENSE"])],
-    install_requires=install_reqs,
-    entry_points={
-        "console_scripts": ["cellxgene-gateway=cellxgene_gateway.gateway:main"]
-    },
-    classifiers=["Topic :: Scientific/Engineering :: Visualization"],
-    python_requires=">=3.6",
 )

tests/items/s3/test_s3item_source.py

@@ -1,7 +1,9 @@
 import unittest
 from unittest.mock import MagicMock, Mock, patch
 
-from cellxgene_gateway.gateway import app
+from flask import Flask
+
+from cellxgene_gateway.gateway_blueprint import gateway_blueprint
 from cellxgene_gateway.items.item import ItemType
 from cellxgene_gateway.items.s3.s3item import S3Item
 from cellxgene_gateway.items.s3.s3item_source import S3ItemSource
@@ -82,6 +84,8 @@ class TestScanDirectory(unittest.TestCase):
 
         s3func.return_value = S3Mock
         source = S3ItemSource("my-bucket")
+        app = Flask(__name__)
+        app.register_blueprint(gateway_blueprint)
         with app.test_request_context(query_string="refresh=true") as test_context:
             tree = source.scan_directory()
 

tests/test_cache_entry.py

@@ -5,7 +5,7 @@ from flask import Flask
 from cellxgene_gateway import flask_util
 from cellxgene_gateway.cache_entry import CacheEntry, CacheEntryStatus
 from cellxgene_gateway.cache_key import CacheKey
-from cellxgene_gateway.gateway import app
+from cellxgene_gateway.gateway_blueprint import gateway_blueprint
 from cellxgene_gateway.items.file.fileitem import FileItem
 from cellxgene_gateway.items.file.fileitem_source import FileItemSource
 from cellxgene_gateway.items.item import ItemType
@@ -18,7 +18,8 @@ key = CacheKey(
 
 class TestRenderEntry(unittest.TestCase):
     def setUp(self):
-        self.app = app
+        self.app = Flask(__name__)
+        self.app.register_blueprint(gateway_blueprint)
         self.app_context = self.app.test_request_context()
         self.app_context.push()
         self.client = self.app.test_client()

tests/test_prune_process_cache.py

@@ -15,7 +15,7 @@ key = CacheKey(
 
 class TestPruneProcessCache(unittest.TestCase):
     @patch("cellxgene_gateway.util.current_time_stamp", new=lambda: 0)
-    @patch("cellxgene_gateway.env.ttl", new="10")
+    @patch("cellxgene_gateway.env.expire_seconds", new=10)
     @patch("cellxgene_gateway.cache_entry.CacheEntry")
     @patch("cellxgene_gateway.cache_entry.CacheEntry")
     def test_GIVEN_one_old_one_new_THEN_prune_old(self, old, new):