From 0f79e1a010fd0a9c89af8df4b3a1b7e09361a529 Mon Sep 17 00:00:00 2001 From: Bob Belnap Date: Mon, 1 Aug 2022 09:45:47 -0400 Subject: [PATCH] Moved prometheus monitoring into flask blueprint --- README.md | 15 + cellxgene_gateway/flask_util.py | 4 +- cellxgene_gateway/gateway.py | 298 ++---------------- cellxgene_gateway/gateway_blueprint.py | 266 ++++++++++++++++ cellxgene_gateway/prometheus.py | 9 + cellxgene_gateway/templates/cache_status.html | 2 +- .../templates/cellxgene_error.html | 4 +- cellxgene_gateway/templates/filecrawl.html | 4 +- cellxgene_gateway/templates/index.html | 4 +- cellxgene_gateway/templates/loading.html | 4 +- .../templates/process_error.html | 4 +- environment.yml | 2 + requirements.txt | 1 + setup.py | 4 +- tests/items/s3/test_s3item_source.py | 6 +- tests/test_cache_entry.py | 5 +- 16 files changed, 339 insertions(+), 293 deletions(-) create mode 100644 cellxgene_gateway/gateway_blueprint.py create mode 100644 cellxgene_gateway/prometheus.py diff --git a/README.md b/README.md index 3200042..1cfa58a 100644 --- a/README.md +++ b/README.md @@ -186,6 +186,21 @@ pip install isort flake8 black isort -rc . # rc means recursive, and was deprecated in dev version of isort black . ``` +## Dependency management + +We use the following files for dependency management: + +* environment.yml - specifies a conda environment sufficient to run the packages +* setup.cfg - lists static dependency information, both minimal and extra dependencies +* setup.py - dynamically generated package information +* requirements.txt - simple wrapper invoking setup.py + +For more details, see the folloiwng links: + +* https://towardsdatascience.com/setuptools-python-571e7d5500f2 +* https://towardsdatascience.com/requirements-vs-setuptools-python-ae3ee66e28af + + # Getting Help diff --git a/cellxgene_gateway/flask_util.py b/cellxgene_gateway/flask_util.py index b594479..d4cc1db 100644 --- a/cellxgene_gateway/flask_util.py +++ b/cellxgene_gateway/flask_util.py @@ -26,8 +26,8 @@ def url(endpoint, descriptor, source_name): def view_url(descriptor, source_name): - return url("do_view", descriptor, source_name) + return url("gateway_blueprint.do_view", descriptor, source_name) def relaunch_url(descriptor, source_name): - return url("do_relaunch", descriptor, source_name) + return url("gateway_blueprint.do_relaunch", descriptor, source_name) diff --git a/cellxgene_gateway/gateway.py b/cellxgene_gateway/gateway.py index dfd7950..05f96f8 100644 --- a/cellxgene_gateway/gateway.py +++ b/cellxgene_gateway/gateway.py @@ -1,50 +1,15 @@ -# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed -# under the Apache License, Version 2.0 (the "License"); you may not use -# this file except in compliance with the License. You may obtain a copy -# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless -# required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. See the License for -# the specific language governing permissions and limitations under the License. -# import BaseHTTPServer -import json import logging import os -import urllib.parse -from threading import Lock, Thread -from flask import ( - Flask, - make_response, - redirect, - render_template, - request, - send_from_directory, - url_for, -) +import typer +from flask import Flask from werkzeug.middleware.proxy_fix import ProxyFix -from prometheus_flask_exporter import PrometheusMetrics -from cellxgene_gateway import env, flask_util -from cellxgene_gateway.backend_cache import BackendCache -from cellxgene_gateway.cache_entry import CacheEntryStatus -from cellxgene_gateway.cache_key import CacheKey -from cellxgene_gateway.cellxgene_exception import CellxgeneException -from cellxgene_gateway.extra_scripts import get_extra_scripts -from cellxgene_gateway.filecrawl import render_item_source -from cellxgene_gateway.process_exception import ProcessException -from cellxgene_gateway.prune_process_cache import PruneProcessCache +from cellxgene_gateway import env, flask_util, gateway_blueprint from cellxgene_gateway.util import current_time_stamp -from cellxgene_gateway import __version__ app = Flask(__name__) -metrics = PrometheusMetrics(app) -metrics.info('app_info', 'Application info', version=__version__) - -item_sources = [] -default_item_source = None - def _force_https(app): def wrapper(environ, start_response): @@ -55,14 +20,6 @@ def _force_https(app): return wrapper -def set_no_cache(resp): - resp.headers["Cache-Control"] = "no-cache, no-store, must-revalidate" - resp.headers["Pragma"] = "no-cache" - resp.headers["Expires"] = "0" - resp.headers["Cache-Control"] = "public, max-age=0" - return resp - - app.wsgi_app = _force_https(app.wsgi_app) if ( env.proxy_fix_for > 0 @@ -80,229 +37,8 @@ if ( x_prefix=env.proxy_fix_prefix, ) -cache = BackendCache() - -@app.errorhandler(CellxgeneException) -def handle_invalid_usage(error): - - message = f"{error.http_status} Error : {error.message}" - - return ( - render_template( - "cellxgene_error.html", - extra_scripts=get_extra_scripts(), - message=message, - ), - error.http_status, - ) - - -@app.errorhandler(ProcessException) -def handle_invalid_process(error): - - message = [] - - message.append(error.message) - message.append(f"{error.http_status} Error.") - message.append(f"Stdout: {error.stdout}") - message.append(f"Stderr: {error.stderr}") - - return ( - render_template( - "process_error.html", - extra_scripts=get_extra_scripts(), - message=error.message, - http_status=error.http_status, - stdout=error.stdout, - stderr=error.stderr, - relaunch_url=error.key.relaunch_url(), - annotation_file=error.key.annotation_descriptor, - ), - error.http_status, - ) - - -@app.route("/favicon.ico") -def favicon(): - return send_from_directory( - os.path.join(app.root_path, "static"), - "nibr.ico", - mimetype="image/vnd.microsof.icon", - ) - - -@app.route("/") -def index(): - return render_template( - "index.html", - ip=env.ip, - cellxgene_data=env.cellxgene_data, - extra_scripts=get_extra_scripts(), - ) - - -@app.route("/filecrawl.html") -@app.route("/filecrawl/") -def filecrawl(path=None): - source_name = request.args.get("source") - sources = ( - filter( - lambda x: x.name == urllib.parse.unquote_plus(source_name), - item_sources, - ) - if source_name - else item_sources - ) - # loop all data sources -- - rendered_sources = [ - render_item_source(item_source, path) for item_source in sources - ] # will we need to make this async in the page??? - rendered_html = "\n".join(rendered_sources) - - resp = make_response( - render_template( - "filecrawl.html", - extra_scripts=get_extra_scripts(), - rendered_html=rendered_html, - path=path, - ) - ) - set_no_cache(resp) - return resp - - -entry_lock = Lock() - - -def matching_source(source_name): - if source_name is None: - source_name = default_item_source.name - matching = [i for i in item_sources if i.name == source_name] - if len(matching) != 1: - raise Exception(f"Could not find matching item source {source_name}") - source = matching[0] - return source - - -@app.route( - "/source//view/", - methods=["GET", "PUT", "POST"], -) -@app.route("/view/", methods=["GET", "PUT", "POST"]) -def do_view(path, source_name=None): - source = matching_source(source_name) - match = cache.check_path(source, path) - - if match is None: - lookup = source.lookup(path) - if lookup is None: - raise CellxgeneException( - f"Could not find item for path {path} in source {source.name}", - 404, - ) - key = CacheKey.for_lookup(source, lookup) - print( - f"view path={path}, source_name={source_name}, dataset={key.file_path}, annotation_file= {key.annotation_file_path}, key={key.descriptor}, source={key.source_name}" - ) - with entry_lock: - match = cache.check_entry(key) - if match is None: - uascripts = get_extra_scripts() - match = cache.create_entry(key, uascripts) - - match.timestamp = current_time_stamp() - - if ( - match.status == CacheEntryStatus.loaded - or match.status == CacheEntryStatus.loading - ): - if source.is_authorized(match.key.descriptor): - return match.serve_content(path) - else: - raise CellxgeneException("User not authorized to access this data", 403) - elif match.status == CacheEntryStatus.error: - raise ProcessException.from_cache_entry(match) - - -@app.route("/cache_status", methods=["GET"]) -def do_GET_status(): - return render_template( - "cache_status.html", - entry_list=cache.entry_list, - extra_scripts=get_extra_scripts(), - ) - - -@app.route("/cache_status.json", methods=["GET"]) -def do_GET_status_json(): - return json.dumps( - { - "launchtime": app.launchtime, - "entry_list": [ - { - "dataset": entry.key.dataset, - "annotation_file": entry.key.annotation_file, - "launchtime": entry.launchtime, - "last_access": entry.timestamp, - "status": entry.status, - } - for entry in cache.entry_list - ], - } - ) - - -@app.route("/relaunch/", methods=["GET"]) -def do_relaunch(path): - source_name = request.args.get("source_name") or default_item_source.name - source = matching_source(source_name) - key = CacheKey.for_lookup(source, source.lookup(path)) - match = cache.check_entry(key) - if not match is None: - match.terminate() - return redirect( - key.view_url, - code=302, - ) - - -@app.route("/terminate/", methods=["GET"]) -def do_terminate(path): - source_name = request.args.get("source_name") or default_item_source.name - source = matching_source(source_name) - key = CacheKey.for_lookup(source, source.lookup(path)) - match = cache.check_entry(key) - if not match is None: - match.terminate() - return redirect(url_for("do_GET_status"), code=302) - - -@app.route("/metadata/ip_address", methods=["GET"]) -def ip_address(): - resp = make_response(env.ip) - return set_no_cache(resp) - - -def launch(): - env.validate() - if not item_sources or not len(item_sources): - raise Exception("No data sources specified for Cellxgene Gateway") - - global default_item_source - if default_item_source is None: - default_item_source = item_sources[0] - - pruner = PruneProcessCache(cache) - - background_thread = Thread(target=pruner) - background_thread.start() - - app.launchtime = current_time_stamp() - app.run(host="0.0.0.0", port=env.gateway_port, debug=False) - - -def main(): +def main(prometheus: bool = False): logging.basicConfig( level=env.log_level, format="%(asctime)s:%(name)s:%(levelname)s:%(message)s", @@ -313,19 +49,33 @@ def main(): if cellxgene_bucket is not None: from cellxgene_gateway.items.s3.s3item_source import S3ItemSource - item_sources.append(S3ItemSource(cellxgene_bucket, name="s3")) + gateway_blueprint.item_sources.append(S3ItemSource(cellxgene_bucket, name="s3")) default_item_source = "s3" if cellxgene_data is not None: from cellxgene_gateway.items.file.fileitem_source import FileItemSource - item_sources.append(FileItemSource(cellxgene_data, name="local")) + gateway_blueprint.item_sources.append( + FileItemSource(cellxgene_data, name="local") + ) default_item_source = "local" - if len(item_sources) == 0: + if len(gateway_blueprint.item_sources) == 0: raise Exception("Please specify CELLXGENE_DATA or CELLXGENE_BUCKET") - flask_util.include_source_in_url = len(item_sources) > 1 + flask_util.include_source_in_url = len(gateway_blueprint.item_sources) > 1 - launch() + if prometheus: + from cellxgene_gateway.prometheus import add_metrics + + add_metrics(app) + app.register_blueprint(gateway_blueprint.gateway_blueprint) + gateway_blueprint.launch() + + app.launchtime = current_time_stamp() + app.run(host="0.0.0.0", port=env.gateway_port, debug=False) + + +def run(): + typer.run(main) if __name__ == "__main__": - main() + run() diff --git a/cellxgene_gateway/gateway_blueprint.py b/cellxgene_gateway/gateway_blueprint.py new file mode 100644 index 0000000..66eac09 --- /dev/null +++ b/cellxgene_gateway/gateway_blueprint.py @@ -0,0 +1,266 @@ +# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed +# under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless +# required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the License for +# the specific language governing permissions and limitations under the License. +# import BaseHTTPServer +import json +import os +import urllib.parse +from threading import Lock, Thread + +from flask import ( + Blueprint, + current_app, + make_response, + redirect, + render_template, + request, + send_from_directory, + url_for, +) + +from cellxgene_gateway import env +from cellxgene_gateway.backend_cache import BackendCache +from cellxgene_gateway.cache_entry import CacheEntryStatus +from cellxgene_gateway.cache_key import CacheKey +from cellxgene_gateway.cellxgene_exception import CellxgeneException +from cellxgene_gateway.extra_scripts import get_extra_scripts +from cellxgene_gateway.filecrawl import render_item_source +from cellxgene_gateway.process_exception import ProcessException +from cellxgene_gateway.prune_process_cache import PruneProcessCache +from cellxgene_gateway.util import current_time_stamp + +gateway_blueprint = Blueprint("gateway_blueprint", __name__) +item_sources = [] +default_item_source = None + + +def set_no_cache(resp): + resp.headers["Cache-Control"] = "no-cache, no-store, must-revalidate" + resp.headers["Pragma"] = "no-cache" + resp.headers["Expires"] = "0" + resp.headers["Cache-Control"] = "public, max-age=0" + return resp + + +cache = BackendCache() + + +@gateway_blueprint.errorhandler(CellxgeneException) +def handle_invalid_usage(error): + + message = f"{error.http_status} Error : {error.message}" + + return ( + render_template( + "cellxgene_error.html", + extra_scripts=get_extra_scripts(), + message=message, + ), + error.http_status, + ) + + +@gateway_blueprint.errorhandler(ProcessException) +def handle_invalid_process(error): + + message = [] + + message.append(error.message) + message.append(f"{error.http_status} Error.") + message.append(f"Stdout: {error.stdout}") + message.append(f"Stderr: {error.stderr}") + + return ( + render_template( + "process_error.html", + extra_scripts=get_extra_scripts(), + message=error.message, + http_status=error.http_status, + stdout=error.stdout, + stderr=error.stderr, + relaunch_url=error.key.relaunch_url(), + annotation_file=error.key.annotation_descriptor, + ), + error.http_status, + ) + + +@gateway_blueprint.route("/favicon.ico") +def favicon(): + return send_from_directory( + os.path.join(current_app.root_path, "static"), + "nibr.ico", + mimetype="image/vnd.microsof.icon", + ) + + +@gateway_blueprint.route("/") +def index(): + return render_template( + "index.html", + ip=env.ip, + cellxgene_data=env.cellxgene_data, + extra_scripts=get_extra_scripts(), + ) + + +@gateway_blueprint.route("/filecrawl.html") +@gateway_blueprint.route("/filecrawl/") +def filecrawl(path=None): + source_name = request.args.get("source") + sources = ( + filter( + lambda x: x.name == urllib.parse.unquote_plus(source_name), + item_sources, + ) + if source_name + else item_sources + ) + # loop all data sources -- + rendered_sources = [ + render_item_source(item_source, path) for item_source in sources + ] # will we need to make this async in the page??? + rendered_html = "\n".join(rendered_sources) + + resp = make_response( + render_template( + "filecrawl.html", + extra_scripts=get_extra_scripts(), + rendered_html=rendered_html, + path=path, + ) + ) + set_no_cache(resp) + return resp + + +entry_lock = Lock() + + +def matching_source(source_name): + if source_name is None: + source_name = default_item_source.name + matching = [i for i in item_sources if i.name == source_name] + if len(matching) != 1: + raise Exception(f"Could not find matching item source {source_name}") + source = matching[0] + return source + + +@gateway_blueprint.route( + "/source//view/", + methods=["GET", "PUT", "POST"], +) +@gateway_blueprint.route("/view/", methods=["GET", "PUT", "POST"]) +def do_view(path, source_name=None): + source = matching_source(source_name) + match = cache.check_path(source, path) + + if match is None: + lookup = source.lookup(path) + if lookup is None: + raise CellxgeneException( + f"Could not find item for path {path} in source {source.name}", + 404, + ) + key = CacheKey.for_lookup(source, lookup) + print( + f"view path={path}, source_name={source_name}, dataset={key.file_path}, annotation_file= {key.annotation_file_path}, key={key.descriptor}, source={key.source_name}" + ) + with entry_lock: + match = cache.check_entry(key) + if match is None: + uascripts = get_extra_scripts() + match = cache.create_entry(key, uascripts) + + match.timestamp = current_time_stamp() + + if ( + match.status == CacheEntryStatus.loaded + or match.status == CacheEntryStatus.loading + ): + if source.is_authorized(match.key.descriptor): + return match.serve_content(path) + else: + raise CellxgeneException("User not authorized to access this data", 403) + elif match.status == CacheEntryStatus.error: + raise ProcessException.from_cache_entry(match) + + +@gateway_blueprint.route("/cache_status", methods=["GET"]) +def do_GET_status(): + return render_template( + "cache_status.html", + entry_list=cache.entry_list, + extra_scripts=get_extra_scripts(), + ) + + +@gateway_blueprint.route("/cache_status.json", methods=["GET"]) +def do_GET_status_json(): + return json.dumps( + { + "launchtime": current_app.launchtime, + "entry_list": [ + { + "dataset": entry.key.dataset, + "annotation_file": entry.key.annotation_file, + "launchtime": entry.launchtime, + "last_access": entry.timestamp, + "status": entry.status, + } + for entry in cache.entry_list + ], + } + ) + + +@gateway_blueprint.route("/relaunch/", methods=["GET"]) +def do_relaunch(path): + source_name = request.args.get("source_name") or default_item_source.name + source = matching_source(source_name) + key = CacheKey.for_lookup(source, source.lookup(path)) + match = cache.check_entry(key) + if not match is None: + match.terminate() + return redirect( + key.view_url, + code=302, + ) + + +@gateway_blueprint.route("/terminate/", methods=["GET"]) +def do_terminate(path): + source_name = request.args.get("source_name") or default_item_source.name + source = matching_source(source_name) + key = CacheKey.for_lookup(source, source.lookup(path)) + match = cache.check_entry(key) + if not match is None: + match.terminate() + return redirect(url_for("gateway_blueprint.do_GET_status"), code=302) + + +@gateway_blueprint.route("/metadata/ip_address", methods=["GET"]) +def ip_address(): + resp = make_response(env.ip) + return set_no_cache(resp) + + +def launch(): + env.validate() + if not item_sources or not len(item_sources): + raise Exception("No data sources specified for Cellxgene Gateway") + + global default_item_source + if default_item_source is None: + default_item_source = item_sources[0] + + pruner = PruneProcessCache(cache) + + background_thread = Thread(target=pruner) + background_thread.start() diff --git a/cellxgene_gateway/prometheus.py b/cellxgene_gateway/prometheus.py new file mode 100644 index 0000000..5b14b08 --- /dev/null +++ b/cellxgene_gateway/prometheus.py @@ -0,0 +1,9 @@ +from prometheus_flask_exporter import PrometheusMetrics + +from cellxgene_gateway import __version__ + + +def add_metrics(app): + metrics = PrometheusMetrics(app) + metrics.info("app_info", "Application info", version=__version__) + return metrics diff --git a/cellxgene_gateway/templates/cache_status.html b/cellxgene_gateway/templates/cache_status.html index 7360232..5808e11 100644 --- a/cellxgene_gateway/templates/cache_status.html +++ b/cellxgene_gateway/templates/cache_status.html @@ -61,7 +61,7 @@ {% if entry.status.name == 'loaded' %} + href="{{ url_for('gateway_blueprint.do_terminate', path=entry.key.descriptor, source_name=entry.key.source_name) }}"> terminate {% endif %} diff --git a/cellxgene_gateway/templates/cellxgene_error.html b/cellxgene_gateway/templates/cellxgene_error.html index aea6764..8fd1a66 100644 --- a/cellxgene_gateway/templates/cellxgene_error.html +++ b/cellxgene_gateway/templates/cellxgene_error.html @@ -28,11 +28,11 @@

{{ message }}

- + Please click here to be redirected to the file directory.
- + Please click here to return to the homepage. diff --git a/cellxgene_gateway/templates/filecrawl.html b/cellxgene_gateway/templates/filecrawl.html index e31f54d..1ff2924 100644 --- a/cellxgene_gateway/templates/filecrawl.html +++ b/cellxgene_gateway/templates/filecrawl.html @@ -36,10 +36,10 @@ Navigation: