From c03098af17ed464fb45af4371b016c808d4baa88 Mon Sep 17 00:00:00 2001 From: Alok Saldanha Date: Sat, 21 Dec 2019 11:22:57 -0500 Subject: [PATCH 1/9] #14 enabled new annotations Due to issue with opening annotation files with "-" in the name, this doesn't quite work. --- cellxgene_gateway/backend_cache.py | 9 +-- cellxgene_gateway/cache_entry.py | 66 ++++++++++------- cellxgene_gateway/cache_key.py | 29 ++++++++ cellxgene_gateway/dir_util.py | 47 ++---------- cellxgene_gateway/env.py | 2 + cellxgene_gateway/filecrawl.py | 72 +++++++++++++++++++ cellxgene_gateway/flask_util.py | 5 ++ cellxgene_gateway/gateway.py | 31 ++++---- cellxgene_gateway/path_util.py | 68 ++++++++++++++---- cellxgene_gateway/process_exception.py | 4 +- cellxgene_gateway/subprocess_backend.py | 19 +++-- cellxgene_gateway/templates/cache_status.html | 4 +- cellxgene_gateway/templates/filecrawl.html | 2 - 13 files changed, 252 insertions(+), 106 deletions(-) create mode 100644 cellxgene_gateway/cache_key.py create mode 100644 cellxgene_gateway/filecrawl.py create mode 100644 cellxgene_gateway/flask_util.py diff --git a/cellxgene_gateway/backend_cache.py b/cellxgene_gateway/backend_cache.py index c038101..31dd4a3 100644 --- a/cellxgene_gateway/backend_cache.py +++ b/cellxgene_gateway/backend_cache.py @@ -33,12 +33,12 @@ class BackendCache: contents = self.entry_list return [c.port for c in contents] - def check_entry(self, dataset): + def check_entry(self, key): contents = self.entry_list matches = [ c for c in contents - if c.dataset == dataset and c.status != "terminated" + if c.key.dataset == key.dataset and c.key.annotation_file == key.annotation_file and c.status != "terminated" ] if len(matches) == 0: @@ -51,13 +51,14 @@ class BackendCache: "Found " + str(len(matches)) + " for " + dataset, ) - def create_entry(self, dataset, file_path, scripts): + def create_entry(self, key, scripts): port = 8000 existing_ports = self.get_ports() + while (port in existing_ports) or is_port_in_use(port): port += 1 - entry = CacheEntry.for_dataset(dataset, file_path, port) + entry = CacheEntry.for_key(key, port) background_thread = Thread( target=process_backend.launch, diff --git a/cellxgene_gateway/cache_entry.py b/cellxgene_gateway/cache_entry.py index 56c9616..bc45a78 100644 --- a/cellxgene_gateway/cache_entry.py +++ b/cellxgene_gateway/cache_entry.py @@ -15,13 +15,13 @@ from requests import get, post, put from cellxgene_gateway import env from cellxgene_gateway.cellxgene_exception import CellxgeneException from cellxgene_gateway.util import current_time_stamp +from cellxgene_gateway.flask_util import querystring class CacheEntry: def __init__( self, pid, - dataset, - file_path, + key, port, launchtime, timestamp, @@ -32,8 +32,7 @@ class CacheEntry: http_status, ): self.pid = pid - self.dataset = dataset - self.file_path = file_path + self.key = key self.port = port self.launchtime = launchtime self.timestamp = timestamp @@ -44,11 +43,11 @@ class CacheEntry: self.http_status = http_status @classmethod - def for_dataset(cls, dataset, file_path, port): + def for_key(cls, key, port): + return cls( None, - dataset, - file_path, + key, port, current_time_stamp(), current_time_stamp(), @@ -93,45 +92,59 @@ class CacheEntry: self.status = "terminated" def serve_content(self, path): - dataset = self.dataset + dataset = self.key.dataset gateway_basepath = ( - f"{env.external_protocol}://{env.external_host}/view/{dataset}/" + f"{env.external_protocol}://{env.external_host}/view/{self.key.pathpart}/" ) - subpath = path[len(dataset) :] # noqa: E203 + subpath = path[len(self.key.pathpart) :] # noqa: E203 if len(subpath) == 0: r = make_response(f"Redirect to {gateway_basepath}\n", 301) - r.headers["location"] = gateway_basepath + r.headers["location"] = gateway_basepath+querystring() return r port = self.port cellxgene_basepath = f"http://127.0.0.1:{port}" - headers = {} + copy_headers = [ + 'accept', + 'accept-encoding', + 'accept-language', + 'cache-control', + 'connection', + 'content-length', + 'content-type', + 'cookie', + 'host', + 'origin', + 'pragma', + 'referer', + 'sec-fetch-mode', + 'sec-fetch-site', + 'user-agent' + ] + for h in copy_headers: + if h in request.headers: + headers[h] = request.headers[h] - if "accept" in request.headers: - headers["accept"] = request.headers["accept"] - if "user-agent" in request.headers: - headers["user-agent"] = request.headers["user-agent"] - if "content-type" in request.headers: - headers["content-type"] = request.headers["content-type"] + full_path = cellxgene_basepath + subpath + querystring() if request.method in ["GET", "HEAD", "OPTIONS"]: cellxgene_response = get( - cellxgene_basepath + subpath, headers=headers + full_path, headers=headers ) elif request.method == "PUT": cellxgene_response = put( - cellxgene_basepath + subpath, + full_path, headers=headers, - data=request.data.decode(), + data=request.data, ) elif request.method == "POST": cellxgene_response = post( - cellxgene_basepath + subpath, + full_path, headers=headers, - data=request.data.decode(), + data=request.data, ) else: raise CellxgeneException( @@ -146,10 +159,15 @@ class CacheEntry: else: gateway_content = cellxgene_response.content + resp_headers = {} + for h in copy_headers: + if h in cellxgene_response.headers: + resp_headers[h] = cellxgene_response.headers[h] + gateway_response = make_response( gateway_content, cellxgene_response.status_code, - {"Content-Type": content_type}, + resp_headers, ) return gateway_response diff --git a/cellxgene_gateway/cache_key.py b/cellxgene_gateway/cache_key.py new file mode 100644 index 0000000..72c1810 --- /dev/null +++ b/cellxgene_gateway/cache_key.py @@ -0,0 +1,29 @@ +# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed +# under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless +# required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the License for +# the specific language governing permissions and limitations under the License. + +import os + +from flask_api import status + +from cellxgene_gateway import env +from cellxgene_gateway.cellxgene_exception import CellxgeneException + +# There are three kinds of CacheKey: +# 1) somedir/dataset.h5ad: a dataset +# in this case, pathpart == dataset == 'somedir/dataset.h5ad' +# 2) somedir/dataset_annotations/saldaal1-T5HMVBNV.csv : an actual annotaitons file. +# in this case, pathpart == 'dataset_annotations/saldaal1-T5HMVBNV.csv', dataset == 'somedir/dataset.h5ad' +# 3) somedir/dataset_annotations: an annotation directory. The corresponding h5ad must exist, but the directory may not. +# in this case, pathpart == 'dataset_annotations', dataset == 'somedir/dataset.h5ad' + +class CacheKey: + def __init__(self, pathpart, dataset, annotation_file): + self.pathpart = pathpart + self.dataset = dataset + self.annotation_file = annotation_file diff --git a/cellxgene_gateway/dir_util.py b/cellxgene_gateway/dir_util.py index 05db382..4ddc1c7 100644 --- a/cellxgene_gateway/dir_util.py +++ b/cellxgene_gateway/dir_util.py @@ -51,45 +51,8 @@ def create_dir(parent_path, dir_name): else: os.mkdir(full_path) - -def recurse_dir(path): - if not os.path.exists(path): - raise CellxgeneException( - "The given path does not exist.", status.HTTP_400_BAD_REQUEST - ) - - def make_entry(el): - full_path = os.path.join(path, el) - if os.path.isfile(full_path): - return { - "path": full_path.replace(env.cellxgene_data, ""), - "name": el, - "type": "file", - } - elif os.path.isdir(full_path): - return { - "path": full_path.replace(env.cellxgene_data, ""), - "name": el, - "type": "directory", - "children": recurse_dir(full_path), - } - else: - raise CellxgeneException( - "Given path is neither file nor directory.", - status.HTTP_400_BAD_REQUEST, - ) - - return [make_entry(x) for x in os.listdir(path)] - - -def render_entries(entries): - return "" - - -def render_entry(entry): - if entry["type"] == "file": - url = f"/view/{entry['path'].lstrip('/')}" - return f"
  • {entry['name']}
    • " - elif entry["type"] == "directory": - url = f"/filecrawl/{entry['path'].lstrip('/')}" - return f"
  • {entry['name']}{render_entries(entry['children'])}
    • " +annotations_suffix = '_annotations' +def make_h5ad(el): + return el[:-len(annotations_suffix)]+'.h5ad' +def make_annotations(el): + return el[:-5]+annotations_suffix diff --git a/cellxgene_gateway/env.py b/cellxgene_gateway/env.py index cf397a1..5e8a6de 100644 --- a/cellxgene_gateway/env.py +++ b/cellxgene_gateway/env.py @@ -20,6 +20,7 @@ ip = os.environ.get("GATEWAY_IP") extra_scripts = os.environ.get("GATEWAY_EXTRA_SCRIPTS") ttl = os.environ.get("GATEWAY_TTL") enable_upload = os.environ.get("GATEWAY_ENABLE_UPLOAD", "").lower() in ['true', '1'] +enable_annotations = os.environ.get("GATEWAY_ENABLE_ANNOTATIONS", "").lower() in ['true', '1'] env_vars = { "CELLXGENE_LOCATION": cellxgene_location, @@ -34,6 +35,7 @@ optional_env_vars = { "GATEWAY_EXTRA_SCRIPTS": extra_scripts, "GATEWAY_TTL": ttl, "GATEWAY_ENABLE_UPLOAD": enable_upload, + "GATEWAY_ENABLE_ANNOTATIONS": enable_annotations, } def validate(): diff --git a/cellxgene_gateway/filecrawl.py b/cellxgene_gateway/filecrawl.py new file mode 100644 index 0000000..7097168 --- /dev/null +++ b/cellxgene_gateway/filecrawl.py @@ -0,0 +1,72 @@ +import os +from cellxgene_gateway import env +from cellxgene_gateway.dir_util import make_h5ad, make_annotations, annotations_suffix + +def recurse_dir(path): + if not os.path.exists(path): + raise CellxgeneException( + "The given path does not exist.", status.HTTP_400_BAD_REQUEST + ) + + all_entries = os.listdir(path) + def is_h5ad(el): + return el.endswith('.h5ad') and os.path.isfile(os.path.join(path, el)) + h5ad_entries = [x for x in all_entries if is_h5ad(x)] + annotation_dir_entries = [x for x in all_entries if x.endswith(annotations_suffix) and make_h5ad(x) in h5ad_entries] + def list_annotations(el): + full_path = os.path.join(path, el) + if not os.path.isdir(full_path): + entries = [] + else: + entries = [{ + "name": x[:x.index('-')] if '-' in x else x, + "path": os.path.join(full_path, x).replace(env.cellxgene_data, ""), + } for x in os.listdir(full_path) if x.endswith('.csv') and os.path.isfile(os.path.join(full_path, x))] + return [{"name":'new', "path":full_path.replace(env.cellxgene_data, "")}] + entries + + def make_entry(el): + full_path = os.path.join(path, el) + if el in h5ad_entries: + return { + "path": full_path.replace(env.cellxgene_data, ""), + "name": el, + "type": "file", + "annotations": list_annotations(make_annotations(el)), + } + elif os.path.isdir(full_path) and el not in annotation_dir_entries: + return { + "path": full_path.replace(env.cellxgene_data, ""), + "name": el, + "type": "directory", + "children": recurse_dir(full_path), + } + else: + return { + "path": full_path, + "name": el, + "type": "neither", + } + + return [make_entry(x) for x in os.listdir(path)] + + +def render_entries(entries): + return "" + +def get_url(entry): + return f"view/{ entry['path'].lstrip('/') }" + +def render_annotations(entry): + if len(entry['annotations']) > 0: + return ' | annotations: ' + ", ".join([f"{a['name']}" for a in entry['annotations']]) + else: + return ''; + +def render_entry(entry): + if entry["type"] == "file": + return f"
  • {entry['name']} {render_annotations(entry)}
    • " + elif entry["type"] == "directory": + url = f"/filecrawl/{entry['path'].lstrip('/')}" + return f"
  • {entry['name']}{render_entries(entry['children'])}
    • " + else: + return "" diff --git a/cellxgene_gateway/flask_util.py b/cellxgene_gateway/flask_util.py new file mode 100644 index 0000000..9727220 --- /dev/null +++ b/cellxgene_gateway/flask_util.py @@ -0,0 +1,5 @@ +from flask import request + +def querystring(): + qs = request.query_string.decode() + return f'?{qs}' if len(qs) > 0 else '' diff --git a/cellxgene_gateway/gateway.py b/cellxgene_gateway/gateway.py index 28187e2..441cdd9 100644 --- a/cellxgene_gateway/gateway.py +++ b/cellxgene_gateway/gateway.py @@ -28,12 +28,13 @@ from werkzeug import secure_filename from cellxgene_gateway import env from cellxgene_gateway.backend_cache import BackendCache from cellxgene_gateway.cellxgene_exception import CellxgeneException -from cellxgene_gateway.dir_util import create_dir, recurse_dir, render_entries, is_subdir +from cellxgene_gateway.dir_util import create_dir, is_subdir +from cellxgene_gateway.filecrawl import recurse_dir, render_entries from cellxgene_gateway.extra_scripts import get_extra_scripts -from cellxgene_gateway.path_util import get_dataset, get_file_path from cellxgene_gateway.process_exception import ProcessException from cellxgene_gateway.prune_process_cache import PruneProcessCache from cellxgene_gateway.util import current_time_stamp +from cellxgene_gateway.path_util import get_key app = Flask(__name__) cache = BackendCache() @@ -73,7 +74,8 @@ def handle_invalid_process(error): http_status=error.http_status, stdout=error.stdout, stderr=error.stderr, - dataset=error.dataset, + dataset=error.key.dataset, + annotation_file=error.key.annotation_file, ), error.http_status, ) @@ -159,7 +161,6 @@ if env.enable_upload: @app.route("/filecrawl.html") def filecrawl(): - entries = recurse_dir(env.cellxgene_data) rendered_html = render_entries(entries) return render_template( @@ -187,13 +188,13 @@ def do_filecrawl(path): entry_lock = Lock() @app.route("/view/", methods=["GET", "PUT", "POST"]) def do_view(path): - dataset = get_dataset(path) - file_path = get_file_path(dataset) + key = get_key(path) + print(f"view path={path}, dataset={key.dataset}, annotation_file= {key.annotation_file}, key={key.pathpart}") with entry_lock: - match = cache.check_entry(dataset) + match = cache.check_entry(key) if match is None: uascripts = get_extra_scripts() - match = cache.create_entry(dataset, file_path, uascripts) + match = cache.create_entry(key, uascripts) match.timestamp = current_time_stamp() @@ -216,7 +217,8 @@ def do_GET_status(): def do_GET_status_json(): return json.dumps({'launchtime':app.launchtime, 'entry_list':[{ - 'dataset': entry.dataset, + 'dataset': entry.key.dataset, + 'annotation_file': entry.key.annotation_file, 'launchtime': entry.launchtime, 'last_access': entry.timestamp, 'status': entry.status @@ -224,16 +226,17 @@ def do_GET_status_json(): @app.route("/relaunch/", methods=["GET"]) def do_relaunch(path): - dataset = get_dataset(path) - match = cache.check_entry(dataset) + key = get_key(path) + match = cache.check_entry(key) if not match is None: match.terminate() - return redirect(url_for("do_view", path=path), code=302) + qs = request.query_string.decode() + return redirect(url_for("do_view", path=path) + (f'?{qs}' if len(qs) > 0 else ''), code=302) @app.route("/terminate/", methods=["GET"]) def do_terminate(path): - dataset = get_dataset(path) - match = cache.check_entry(dataset) + key = get_key(path) + match = cache.check_entry(key) if not match is None: match.terminate() return redirect(url_for("do_GET_status"), code=302) diff --git a/cellxgene_gateway/path_util.py b/cellxgene_gateway/path_util.py index 6dc4fb6..cf93d1c 100644 --- a/cellxgene_gateway/path_util.py +++ b/cellxgene_gateway/path_util.py @@ -13,37 +13,81 @@ from flask_api import status from cellxgene_gateway import env from cellxgene_gateway.cellxgene_exception import CellxgeneException +from cellxgene_gateway.dir_util import make_h5ad +from cellxgene_gateway.cache_key import CacheKey - -def get_dataset(path): +def get_key(path): if path == "/" or path == "": raise CellxgeneException( "No matching dataset found.", status.HTTP_404_NOT_FOUND ) trimmed = path[:-1] if path[-1] == "/" else path - try: - get_file_path(trimmed) - return trimmed + # valid paths come in three forms: + if trimmed.endswith('.h5ad') and data_file_exists(trimmed): + # 1) somedir/dataset.h5ad: a dataset + return CacheKey(trimmed, trimmed, None) + elif trimmed.endswith('.csv') and data_file_exists(trimmed): + + # 2) somedir/dataset_annotations/saldaal1-T5HMVBNV.csv : an actual annotaitons file. + annotations_dir = os.path.split(trimmed)[0] + dataset = make_h5ad(annotations_dir) + if data_file_exists(dataset): + return CacheKey(trimmed, dataset, trimmed) + elif trimmed.endswith('_annotations') and data_dir_exists(trimmed): + # 3) somedir/dataset_annotations: an annotation directory. The corresponding h5ad must exist, but the directory may not. + dataset = make_h5ad(trimmed) + if data_file_exists(dataset): + return CacheKey(trimmed, dataset, '') except CellxgeneException: - split = os.path.split(trimmed) - return get_dataset(split[0]) + pass + split = os.path.split(trimmed) + return get_key(split[0]) - -def validate_path(file_path): +def validate_exists(file_path): if not os.path.exists(file_path): raise CellxgeneException( "File does not exist: " + file_path, status.HTTP_400_BAD_REQUEST ) + +def validate_is_file(file_path): + validate_exists(file_path) if not os.path.isfile(file_path): raise CellxgeneException( "Path is not file: " + file_path, status.HTTP_400_BAD_REQUEST ) return +def validate_is_dir(file_path): + validate_exists(file_path) + if not os.path.isdir(file_path): + raise CellxgeneException( + "Path is not dir: " + file_path, status.HTTP_400_BAD_REQUEST + ) + return - -def get_file_path(dataset): +def data_file_exists(dataset): file_path = os.path.join(env.cellxgene_data, dataset) - validate_path(file_path) + validate_is_file(file_path) + return True +def data_dir_exists(dataset): + file_path = os.path.join(env.cellxgene_data, dataset) + validate_is_dir(file_path) + return True + +def get_file_path(key): + dataset = key.dataset + file_path = os.path.join(env.cellxgene_data, dataset) + validate_is_file(file_path) + return file_path + +def get_annotation_file_path(key): + print(f"getting annotaiton_file_path for {key.annotation_file}") + if key.annotation_file is None: + return None + if key.annotation_file == '': + return '' + file_path = os.path.join(env.cellxgene_data, key.annotation_file) + print(f"getting annotaiton_file_path for {key}, file_path {file_path}") + validate_is_file(file_path) return file_path diff --git a/cellxgene_gateway/process_exception.py b/cellxgene_gateway/process_exception.py index a90f35d..6e6ea36 100644 --- a/cellxgene_gateway/process_exception.py +++ b/cellxgene_gateway/process_exception.py @@ -15,7 +15,7 @@ class ProcessException(Exception): self.stdout = stdout self.stderr = stderr self.http_status = http_status - self.dataset = dataset + self.key = key @classmethod def from_cache_entry(cls, cache_entry): @@ -24,5 +24,5 @@ class ProcessException(Exception): cache_entry.all_output, cache_entry.stderr, cache_entry.http_status, - cache_entry.dataset, + cache_entry.key, ) diff --git a/cellxgene_gateway/subprocess_backend.py b/cellxgene_gateway/subprocess_backend.py index 3948da0..b2d6a36 100644 --- a/cellxgene_gateway/subprocess_backend.py +++ b/cellxgene_gateway/subprocess_backend.py @@ -11,21 +11,30 @@ import logging import subprocess from flask_api import status - +from cellxgene_gateway.env import enable_annotations from cellxgene_gateway.process_exception import ProcessException - +from cellxgene_gateway.dir_util import make_annotations +from cellxgene_gateway.path_util import get_file_path, get_annotation_file_path class SubprocessBackend: def __init__(self): pass - def create_cmd(self, cellxgene_loc, file_path, port, scripts): - + def create_cmd(self, cellxgene_loc, file_path, port, scripts, annotation_file_path): + if enable_annotations and not annotation_file_path is None: + annotation_args_prefix = " --experimental-annotations" + if annotation_file_path == "": + annotation_args = f"{annotation_args_prefix} --experimental-annotations-output-dir {make_annotations(file_path)}" + else: + annotation_args = f"{annotation_args_prefix} --experimental-annotations-file {annotation_file_path}" + else: + annotation_args = "" cmd = ( f"yes | {cellxgene_loc} launch {file_path}" + " --port " + str(port) + " --host 127.0.0.1" + + annotation_args ) for s in scripts: @@ -36,7 +45,7 @@ class SubprocessBackend: def launch(self, cellxgene_loc, scripts, cache_entry): cmd = self.create_cmd( - cellxgene_loc, cache_entry.file_path, cache_entry.port, scripts + cellxgene_loc, get_file_path(cache_entry.key), cache_entry.port, scripts, get_annotation_file_path(cache_entry.key) ) logging.getLogger("cellxgene_gateway").info(f"launching {cmd}") process = subprocess.Popen( diff --git a/cellxgene_gateway/templates/cache_status.html b/cellxgene_gateway/templates/cache_status.html index aad3813..38bb204 100644 --- a/cellxgene_gateway/templates/cache_status.html +++ b/cellxgene_gateway/templates/cache_status.html @@ -29,6 +29,7 @@ PID dataset + annotation_file port launchtime last access @@ -42,7 +43,8 @@ {% for entry in entry_list %} {{ entry.pid }} - {{ entry.dataset }} + {{ entry.key.dataset }} + {{ entry.key.annotation_file }} {{ entry.port }} {{ entry.launchtime }} {{ entry.timestamp }} diff --git a/cellxgene_gateway/templates/filecrawl.html b/cellxgene_gateway/templates/filecrawl.html index d752163..a6a5948 100644 --- a/cellxgene_gateway/templates/filecrawl.html +++ b/cellxgene_gateway/templates/filecrawl.html @@ -27,11 +27,9 @@

    Cellxgene Gateway - FILE CRAWLER

    {% endif %} -

    Please click on a dataset to view it in Cellxgene Server.

    -
    {{ rendered_html|safe }}

    Navigation: From 736541c4ec46361b40c8b0b107fe94cc8668e19e Mon Sep 17 00:00:00 2001 From: Alok Saldanha Date: Sat, 28 Dec 2019 22:11:15 -0500 Subject: [PATCH 2/9] #14 Added support for listing files with _ or - as separator --- cellxgene_gateway/filecrawl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cellxgene_gateway/filecrawl.py b/cellxgene_gateway/filecrawl.py index 7097168..3b0234c 100644 --- a/cellxgene_gateway/filecrawl.py +++ b/cellxgene_gateway/filecrawl.py @@ -19,7 +19,8 @@ def recurse_dir(path): entries = [] else: entries = [{ - "name": x[:x.index('-')] if '-' in x else x, + "name": x[:-13] if (x[-13] in ['-','_']) else ( + x[:-4] if x.endwith('.csv') else x), "path": os.path.join(full_path, x).replace(env.cellxgene_data, ""), } for x in os.listdir(full_path) if x.endswith('.csv') and os.path.isfile(os.path.join(full_path, x))] return [{"name":'new', "path":full_path.replace(env.cellxgene_data, "")}] + entries From 201f87634143dc39957d7194932ca1eb6e842d95 Mon Sep 17 00:00:00 2001 From: Alok Saldanha Date: Wed, 1 Jan 2020 01:12:06 -0500 Subject: [PATCH 3/9] #14 added custom method to create data dirs --- cellxgene_gateway/filecrawl.py | 8 +++++--- cellxgene_gateway/gateway.py | 10 ++++++++-- cellxgene_gateway/path_util.py | 12 +++++++----- cellxgene_gateway/static/js/annotation.js | 21 +++++++++++++++++++++ cellxgene_gateway/templates/filecrawl.html | 8 +++++++- 5 files changed, 48 insertions(+), 11 deletions(-) create mode 100644 cellxgene_gateway/static/js/annotation.js diff --git a/cellxgene_gateway/filecrawl.py b/cellxgene_gateway/filecrawl.py index 3b0234c..8ce66b7 100644 --- a/cellxgene_gateway/filecrawl.py +++ b/cellxgene_gateway/filecrawl.py @@ -23,7 +23,7 @@ def recurse_dir(path): x[:-4] if x.endwith('.csv') else x), "path": os.path.join(full_path, x).replace(env.cellxgene_data, ""), } for x in os.listdir(full_path) if x.endswith('.csv') and os.path.isfile(os.path.join(full_path, x))] - return [{"name":'new', "path":full_path.replace(env.cellxgene_data, "")}] + entries + return [{"name":'new', "class":'new', "path":full_path.replace(env.cellxgene_data, "")}] + entries def make_entry(el): full_path = os.path.join(path, el) @@ -56,12 +56,14 @@ def render_entries(entries): def get_url(entry): return f"view/{ entry['path'].lstrip('/') }" +def get_class(entry): + return f" class='{entry['class']}'" if 'class' in entry else '' def render_annotations(entry): if len(entry['annotations']) > 0: - return ' | annotations: ' + ", ".join([f"{a['name']}" for a in entry['annotations']]) + return ' | annotations: ' + ", ".join([f"{a['name']}" for a in entry['annotations']]) else: - return ''; + return '' def render_entry(entry): if entry["type"] == "file": diff --git a/cellxgene_gateway/gateway.py b/cellxgene_gateway/gateway.py index 441cdd9..b3d34ec 100644 --- a/cellxgene_gateway/gateway.py +++ b/cellxgene_gateway/gateway.py @@ -17,6 +17,7 @@ import json from flask import ( Flask, redirect, + make_response, render_template, request, send_from_directory, @@ -163,11 +164,16 @@ if env.enable_upload: def filecrawl(): entries = recurse_dir(env.cellxgene_data) rendered_html = render_entries(entries) - return render_template( + resp = make_response(render_template( "filecrawl.html", extra_scripts=get_extra_scripts(), rendered_html=rendered_html, - ) + )) + resp.headers["Cache-Control"] = "no-cache, no-store, must-revalidate" + resp.headers["Pragma"] = "no-cache" + resp.headers["Expires"] = "0" + resp.headers['Cache-Control'] = 'public, max-age=0' + return resp @app.route("/filecrawl/") def do_filecrawl(path): diff --git a/cellxgene_gateway/path_util.py b/cellxgene_gateway/path_util.py index cf93d1c..08e5ccf 100644 --- a/cellxgene_gateway/path_util.py +++ b/cellxgene_gateway/path_util.py @@ -28,12 +28,13 @@ def get_key(path): if trimmed.endswith('.h5ad') and data_file_exists(trimmed): # 1) somedir/dataset.h5ad: a dataset return CacheKey(trimmed, trimmed, None) - elif trimmed.endswith('.csv') and data_file_exists(trimmed): + elif trimmed.endswith('.csv'): - # 2) somedir/dataset_annotations/saldaal1-T5HMVBNV.csv : an actual annotaitons file. + # 2) somedir/dataset_annotations/saldaal1-T5HMVBNV.csv : an actual annotations file. annotations_dir = os.path.split(trimmed)[0] dataset = make_h5ad(annotations_dir) if data_file_exists(dataset): + data_dir_ensure(annotations_dir) return CacheKey(trimmed, dataset, trimmed) elif trimmed.endswith('_annotations') and data_dir_exists(trimmed): # 3) somedir/dataset_annotations: an annotation directory. The corresponding h5ad must exist, but the directory may not. @@ -74,6 +75,10 @@ def data_dir_exists(dataset): file_path = os.path.join(env.cellxgene_data, dataset) validate_is_dir(file_path) return True +def data_dir_ensure(dataset): + file_path = os.path.join(env.cellxgene_data, dataset) + if not os.path.exists(file_path): + os.makedirs(file_path) def get_file_path(key): dataset = key.dataset @@ -82,12 +87,9 @@ def get_file_path(key): return file_path def get_annotation_file_path(key): - print(f"getting annotaiton_file_path for {key.annotation_file}") if key.annotation_file is None: return None if key.annotation_file == '': return '' file_path = os.path.join(env.cellxgene_data, key.annotation_file) - print(f"getting annotaiton_file_path for {key}, file_path {file_path}") - validate_is_file(file_path) return file_path diff --git a/cellxgene_gateway/static/js/annotation.js b/cellxgene_gateway/static/js/annotation.js new file mode 100644 index 0000000..9698a13 --- /dev/null +++ b/cellxgene_gateway/static/js/annotation.js @@ -0,0 +1,21 @@ +// neandertal javascript +const new_annotation_callback = (() =>{ + const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + const pickn = (c, n) => n==0?'':c.substr(Math.random()*c.length,1) + pickn(c,n-1); + const suffix = `_${pickn(chars,8)}.csv`; + return (e) => { + e.preventDefault(); + const el = $(e.target); + const href = el.attr('href'); + const base = prompt(`Name your annotations collection\nnote: the suffix "${suffix}" will be appended`); + if (base !== null && base.length > 0) { + if (/^[0-9a-zA-Z_]+$/.test(base)) { + window.location = `${href}/${base}${suffix}`; + } else { + alert("Error: name must match ^[0-9a-zA-Z_]+$\nthat is, only numbers, letters and underscore are allowed") + } + } + return false; + } +})() + diff --git a/cellxgene_gateway/templates/filecrawl.html b/cellxgene_gateway/templates/filecrawl.html index a6a5948..7d59e34 100644 --- a/cellxgene_gateway/templates/filecrawl.html +++ b/cellxgene_gateway/templates/filecrawl.html @@ -16,7 +16,8 @@ {% for script in extra_scripts %} - {% endfor %} + {% endfor %} + @@ -41,5 +42,10 @@

  • homepage

    • + From 11efd3a9548a58898300ec55714dd9328e630558 Mon Sep 17 00:00:00 2001 From: Alok Saldanha Date: Sun, 5 Jan 2020 20:13:47 -0500 Subject: [PATCH 4/9] #14 update links for opening and terminating datasets --- cellxgene_gateway/templates/cache_status.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cellxgene_gateway/templates/cache_status.html b/cellxgene_gateway/templates/cache_status.html index 38bb204..34b6f52 100644 --- a/cellxgene_gateway/templates/cache_status.html +++ b/cellxgene_gateway/templates/cache_status.html @@ -43,7 +43,7 @@ {% for entry in entry_list %} {{ entry.pid }} - {{ entry.key.dataset }} + {{ entry.key.dataset }} {{ entry.key.annotation_file }} {{ entry.port }} {{ entry.launchtime }} @@ -53,7 +53,7 @@ {{ entry.http_status }} {% if entry.status == 'loaded' %} - terminate + terminate {% endif %} From 6c611f55ddc7ae0489ca5f1d77a728ceefac43d8 Mon Sep 17 00:00:00 2001 From: Alok Saldanha Date: Tue, 7 Jan 2020 08:50:39 -0500 Subject: [PATCH 5/9] #14 redirect prior to returning loading screen --- cellxgene_gateway/cache_entry.py | 12 ++++++++---- cellxgene_gateway/gateway.py | 8 +------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/cellxgene_gateway/cache_entry.py b/cellxgene_gateway/cache_entry.py index bc45a78..2b1a429 100644 --- a/cellxgene_gateway/cache_entry.py +++ b/cellxgene_gateway/cache_entry.py @@ -8,8 +8,9 @@ # the specific language governing permissions and limitations under the License. import psutil import logging +import datetime -from flask import make_response, request +from flask import make_response, request, render_template from requests import get, post, put from cellxgene_gateway import env @@ -92,17 +93,20 @@ class CacheEntry: self.status = "terminated" def serve_content(self, path): - dataset = self.key.dataset - gateway_basepath = ( f"{env.external_protocol}://{env.external_host}/view/{self.key.pathpart}/" ) subpath = path[len(self.key.pathpart) :] # noqa: E203 - + if len(subpath) == 0: r = make_response(f"Redirect to {gateway_basepath}\n", 301) r.headers["location"] = gateway_basepath+querystring() return r + elif self.status == "loading": + launch_time = datetime.datetime.fromtimestamp(self.launchtime) + return render_template( + "loading.html", launchtime=launch_time, all_output=self.all_output + ) port = self.port cellxgene_basepath = f"http://127.0.0.1:{port}" diff --git a/cellxgene_gateway/gateway.py b/cellxgene_gateway/gateway.py index b3d34ec..272f7cd 100644 --- a/cellxgene_gateway/gateway.py +++ b/cellxgene_gateway/gateway.py @@ -8,7 +8,6 @@ # the specific language governing permissions and limitations under the License. # import BaseHTTPServer -import datetime import os import logging from threading import Thread, Lock @@ -204,13 +203,8 @@ def do_view(path): match.timestamp = current_time_stamp() - if match.status == "loaded": + if match.status == "loaded" or match.status == "loading": return match.serve_content(path) - elif match.status == "loading": - launch_time = datetime.datetime.fromtimestamp(match.launchtime) - return render_template( - "loading.html", launchtime=launch_time, all_output=match.all_output - ) elif match.status == "error": raise ProcessException.from_cache_entry(match) From 9f1d217e508685e1761eece68f38456cf5b5506e Mon Sep 17 00:00:00 2001 From: Alok Saldanha Date: Wed, 8 Jan 2020 03:32:06 -0500 Subject: [PATCH 6/9] #14 properly use the external protocol in 302 redirects --- cellxgene_gateway/gateway.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cellxgene_gateway/gateway.py b/cellxgene_gateway/gateway.py index 272f7cd..e34e181 100644 --- a/cellxgene_gateway/gateway.py +++ b/cellxgene_gateway/gateway.py @@ -37,6 +37,14 @@ from cellxgene_gateway.util import current_time_stamp from cellxgene_gateway.path_util import get_key app = Flask(__name__) + +def _force_https(app): + def wrapper(environ, start_response): + environ['wsgi.url_scheme'] = env.external_protocol + return app(environ, start_response) + return wrapper +app.wsgi_app = _force_https(app.wsgi_app) + cache = BackendCache() location = f"{env.external_protocol}://{env.external_host}" @@ -151,7 +159,7 @@ def upload_file(): "Invalid directory.", status.HTTP_400_BAD_REQUEST ) - return redirect(env.location, code=302) + return redirect(location, code=302) if env.enable_upload: From 525a9691c38c09300eedceb07dd23d728003813f Mon Sep 17 00:00:00 2001 From: Alok Saldanha Date: Sat, 18 Jan 2020 07:57:52 -0500 Subject: [PATCH 7/9] #14 fix bug in pruning code --- cellxgene_gateway/prune_process_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cellxgene_gateway/prune_process_cache.py b/cellxgene_gateway/prune_process_cache.py index ca89788..3bb8f2b 100644 --- a/cellxgene_gateway/prune_process_cache.py +++ b/cellxgene_gateway/prune_process_cache.py @@ -34,7 +34,7 @@ class PruneProcessCache: for process in processes_to_delete: try: - logger.info(f"pruning process {process.pid} ({process.dataset})") + logger.info(f"pruning process {process.pid} ({process.key.dataset})") self.cache.prune(process) except Exception: logger.exception("failed to prune process {process.pid} ({process.dataset})") From 96ac41d8606858879cd28e32edf33e2f96d02803 Mon Sep 17 00:00:00 2001 From: Alok Saldanha Date: Sat, 18 Jan 2020 08:00:19 -0500 Subject: [PATCH 8/9] #14 remove random characters from suffix --- cellxgene_gateway/static/js/annotation.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cellxgene_gateway/static/js/annotation.js b/cellxgene_gateway/static/js/annotation.js index 9698a13..73c1739 100644 --- a/cellxgene_gateway/static/js/annotation.js +++ b/cellxgene_gateway/static/js/annotation.js @@ -1,8 +1,6 @@ // neandertal javascript const new_annotation_callback = (() =>{ - const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; - const pickn = (c, n) => n==0?'':c.substr(Math.random()*c.length,1) + pickn(c,n-1); - const suffix = `_${pickn(chars,8)}.csv`; + const suffix = `.csv`; return (e) => { e.preventDefault(); const el = $(e.target); From cc09ac3b957b9a82cc18cf92d1b3ae014867f637 Mon Sep 17 00:00:00 2001 From: Alok Saldanha Date: Sat, 18 Jan 2020 08:06:38 -0500 Subject: [PATCH 9/9] #14 fixed links in folder listings --- cellxgene_gateway/filecrawl.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cellxgene_gateway/filecrawl.py b/cellxgene_gateway/filecrawl.py index 8ce66b7..57f0c41 100644 --- a/cellxgene_gateway/filecrawl.py +++ b/cellxgene_gateway/filecrawl.py @@ -19,8 +19,8 @@ def recurse_dir(path): entries = [] else: entries = [{ - "name": x[:-13] if (x[-13] in ['-','_']) else ( - x[:-4] if x.endwith('.csv') else x), + "name": x[:-13] if (len(x) > 13 and x[-13] in ['-','_']) else ( + x[:-4] if x.endswith('.csv') else x), "path": os.path.join(full_path, x).replace(env.cellxgene_data, ""), } for x in os.listdir(full_path) if x.endswith('.csv') and os.path.isfile(os.path.join(full_path, x))] return [{"name":'new', "class":'new', "path":full_path.replace(env.cellxgene_data, "")}] + entries @@ -55,7 +55,7 @@ def render_entries(entries): return "
      " + "\n".join([render_entry(e) for e in entries]) + "
    " def get_url(entry): - return f"view/{ entry['path'].lstrip('/') }" + return f"/view/{ entry['path'].lstrip('/') }" def get_class(entry): return f" class='{entry['class']}'" if 'class' in entry else ''