From 5404a8b0f3ca7df3ffb1118335bf22c2c03f812b Mon Sep 17 00:00:00 2001 From: Alok Saldanha Date: Sun, 28 Mar 2021 19:13:50 -0400 Subject: [PATCH] removed trailing slash from render_annotations --- .pre-commit-config.yaml | 6 ------ cellxgene_gateway/cache_entry.py | 16 ++++++---------- cellxgene_gateway/env.py | 8 ++------ cellxgene_gateway/filecrawl.py | 21 +++++++++------------ cellxgene_gateway/gateway.py | 23 ++++++----------------- cellxgene_gateway/prune_process_cache.py | 11 +++-------- cellxgene_gateway/subprocess_backend.py | 15 ++++----------- 7 files changed, 30 insertions(+), 70 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e7e8bae..2ed049d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,12 +7,6 @@ repos: language: system types: [python] stages: [commit] - - id: flake8 - name: flake8 - language: system - entry: flake8 - types: [python] - stages: [commit] - id: black language_version: python3.6+ name: black diff --git a/cellxgene_gateway/cache_entry.py b/cellxgene_gateway/cache_entry.py index ea12390..2eb6cca 100644 --- a/cellxgene_gateway/cache_entry.py +++ b/cellxgene_gateway/cache_entry.py @@ -8,14 +8,14 @@ # the specific language governing permissions and limitations under the License. import datetime import logging -from flask.helpers import url_for -from flask.wrappers import Response +import re +from enum import Enum import psutil -from enum import Enum from flask import make_response, render_template, request +from flask.helpers import url_for +from flask.wrappers import Response from requests import get, post, put -import re from cellxgene_gateway import env from cellxgene_gateway.cellxgene_exception import CellxgeneException @@ -103,9 +103,7 @@ class CacheEntry: terminated.append(p.pid) p.terminate() psutil.wait_procs([p], callback=on_terminate) - logging.getLogger("cellxgene_gateway").info( - f"terminated {terminated}" - ) + logging.getLogger("cellxgene_gateway").info(f"terminated {terminated}") self.status = CacheEntryStatus.terminated def rewrite_text_content(self, cellxgene_content): @@ -182,9 +180,7 @@ class CacheEntry: data=request.data, ) else: - raise CellxgeneException( - f"Unexpected method {request.method}", 400 - ) + raise CellxgeneException(f"Unexpected method {request.method}", 400) content_type = cellxgene_response.headers["content-type"] if "text" in content_type: gateway_content = self.rewrite_text_content( diff --git a/cellxgene_gateway/env.py b/cellxgene_gateway/env.py index 2cc491a..b7a98fd 100644 --- a/cellxgene_gateway/env.py +++ b/cellxgene_gateway/env.py @@ -29,15 +29,11 @@ enable_upload = os.environ.get("GATEWAY_ENABLE_UPLOAD", "").lower() in [ "true", "1", ] -enable_annotations = os.environ.get( - "GATEWAY_ENABLE_ANNOTATIONS", "" -).lower() in [ +enable_annotations = os.environ.get("GATEWAY_ENABLE_ANNOTATIONS", "").lower() in [ "true", "1", ] -enable_backed_mode = os.environ.get( - "GATEWAY_ENABLE_BACKED_MODE", "" -).lower() in [ +enable_backed_mode = os.environ.get("GATEWAY_ENABLE_BACKED_MODE", "").lower() in [ "true", "1", ] diff --git a/cellxgene_gateway/filecrawl.py b/cellxgene_gateway/filecrawl.py index 97c2126..5a96f9d 100644 --- a/cellxgene_gateway/filecrawl.py +++ b/cellxgene_gateway/filecrawl.py @@ -8,14 +8,12 @@ # the specific language governing permissions and limitations under the License. import os -from cellxgene_gateway import env -from cellxgene_gateway.dir_util import ( - make_h5ad, - make_annotations, - annotations_suffix, -) + from flask import url_for +from cellxgene_gateway import env +from cellxgene_gateway.dir_util import annotations_suffix, make_annotations, make_h5ad + def recurse_dir(path): if not os.path.exists(path): @@ -45,19 +43,18 @@ def recurse_dir(path): "name": x[:-13] if (len(x) > 13 and x[-13] in ["-", "_"]) else (x[:-4] if x.endswith(".csv") else x), - "path": os.path.join(full_path, x).replace( + "path": os.path.join(full_path, x + "/").replace( env.cellxgene_data, "" ), } for x in sorted(os.listdir(full_path)) - if x.endswith(".csv") - and os.path.isfile(os.path.join(full_path, x)) + if x.endswith(".csv") and os.path.isfile(os.path.join(full_path, x)) ] return [ { "name": "new", "class": "new", - "path": full_path.replace(env.cellxgene_data, ""), + "path": full_path.replace(env.cellxgene_data, "").rstrip("/"), } ] + entries @@ -92,7 +89,7 @@ def render_entries(entries): def get_url(entry): - return url_for("do_view", path=entry["path"].lstrip("/") + "/") + return url_for("do_view", path=entry["path"].lstrip("/")) def get_class(entry): @@ -113,7 +110,7 @@ def render_annotations(entry): def render_entry(entry): if entry["type"] == "file": - return f"
  • {entry['name']} {render_annotations(entry)}
    • " + return f"
  • {entry['name']} {render_annotations(entry)}
    • " elif entry["type"] == "directory": url = f"/filecrawl/{entry['path'].lstrip('/')}" return f"
  • {entry['name']}{render_entries(entry['children'])}
    • " diff --git a/cellxgene_gateway/gateway.py b/cellxgene_gateway/gateway.py index 12e9352..08a6f10 100644 --- a/cellxgene_gateway/gateway.py +++ b/cellxgene_gateway/gateway.py @@ -9,8 +9,6 @@ import json import logging - -# import BaseHTTPServer import os from threading import Lock, Thread @@ -24,8 +22,9 @@ from flask import ( url_for, ) from flask_api import status -from werkzeug.utils import secure_filename from werkzeug.middleware.proxy_fix import ProxyFix +from werkzeug.utils import secure_filename + from cellxgene_gateway import env from cellxgene_gateway.backend_cache import BackendCache from cellxgene_gateway.cache_entry import CacheEntryStatus @@ -163,11 +162,7 @@ def upload_file(): if "file" in request.files: f = request.files["file"] if f and f.filename.endswith(".h5ad"): - f.save( - os.path.join( - full_upload_path, secure_filename(f.filename) - ) - ) + f.save(os.path.join(full_upload_path, secure_filename(f.filename))) return redirect(url_for("filecrawl"), code=302) else: raise CellxgeneException( @@ -180,21 +175,15 @@ def upload_file(): status.HTTP_400_BAD_REQUEST, ) else: - raise CellxgeneException( - "Invalid directory.", status.HTTP_400_BAD_REQUEST - ) + raise CellxgeneException("Invalid directory.", status.HTTP_400_BAD_REQUEST) return redirect(url_for("index"), code=302) if env.enable_upload: app.add_url_rule("/make_user", "make_user", make_user, methods=["POST"]) - app.add_url_rule( - "/make_subdir", "make_subdir", make_subdir, methods=["POST"] - ) - app.add_url_rule( - "/upload_file", "upload_file", upload_file, methods=["POST"] - ) + app.add_url_rule("/make_subdir", "make_subdir", make_subdir, methods=["POST"]) + app.add_url_rule("/upload_file", "upload_file", upload_file, methods=["POST"]) def set_no_cache(resp): diff --git a/cellxgene_gateway/prune_process_cache.py b/cellxgene_gateway/prune_process_cache.py index 98126ec..d4b2f2e 100644 --- a/cellxgene_gateway/prune_process_cache.py +++ b/cellxgene_gateway/prune_process_cache.py @@ -10,8 +10,7 @@ import logging import time -from cellxgene_gateway import env -from cellxgene_gateway import util +from cellxgene_gateway import env, util logger = logging.getLogger(__name__) @@ -29,9 +28,7 @@ class PruneProcessCache: def prune(self): timestamp = util.current_time_stamp() cutoff = timestamp - self.expire_seconds - processes_to_delete = [ - p for p in self.cache.entry_list if p.timestamp < cutoff - ] + processes_to_delete = [p for p in self.cache.entry_list if p.timestamp < cutoff] processes_to_keep = [ p for p in self.cache.entry_list if not p.timestamp < cutoff ] @@ -42,9 +39,7 @@ class PruneProcessCache: for process in processes_to_delete: try: - logger.info( - f"pruning process {process.pid} ({process.key.dataset})" - ) + logger.info(f"pruning process {process.pid} ({process.key.dataset})") self.cache.prune(process) except Exception: logger.exception( diff --git a/cellxgene_gateway/subprocess_backend.py b/cellxgene_gateway/subprocess_backend.py index dffdf9c..7e67959 100644 --- a/cellxgene_gateway/subprocess_backend.py +++ b/cellxgene_gateway/subprocess_backend.py @@ -11,14 +11,11 @@ import logging import subprocess from flask_api import status + from cellxgene_gateway.cache_entry import CacheEntryStatus from cellxgene_gateway.dir_util import make_annotations +from cellxgene_gateway.env import cellxgene_args, enable_annotations, enable_backed_mode from cellxgene_gateway.path_util import get_annotation_file_path, get_file_path -from cellxgene_gateway.env import ( - enable_annotations, - enable_backed_mode, - cellxgene_args, -) from cellxgene_gateway.process_exception import ProcessException @@ -26,14 +23,10 @@ class SubprocessBackend: def __init__(self): pass - def create_cmd( - self, cellxgene_loc, file_path, port, scripts, annotation_file_path - ): + def create_cmd(self, cellxgene_loc, file_path, port, scripts, annotation_file_path): if enable_annotations and not annotation_file_path is None: if annotation_file_path == "": - extra_args = ( - f" --annotations-dir {make_annotations(file_path)}" - ) + extra_args = f" --annotations-dir {make_annotations(file_path)}" else: extra_args = f" --annotations-file {annotation_file_path}" else: