diff --git a/cellxgene_gateway/backend_cache.py b/cellxgene_gateway/backend_cache.py index 75bd352..df20eb2 100644 --- a/cellxgene_gateway/backend_cache.py +++ b/cellxgene_gateway/backend_cache.py @@ -13,7 +13,7 @@ from threading import Thread from flask_api import status from cellxgene_gateway import env -from cellxgene_gateway.cache_entry import CacheEntry +from cellxgene_gateway.cache_entry import CacheEntry, CacheEntryStatus from cellxgene_gateway.cellxgene_exception import CellxgeneException from cellxgene_gateway.subprocess_backend import SubprocessBackend @@ -42,7 +42,7 @@ class BackendCache: for c in contents if c.key.dataset == key.dataset and c.key.annotation_file == key.annotation_file - and c.status != "terminated" + and c.status != CacheEntryStatus.terminated ] if len(matches) == 0: diff --git a/cellxgene_gateway/cache_entry.py b/cellxgene_gateway/cache_entry.py index b032085..5520a68 100644 --- a/cellxgene_gateway/cache_entry.py +++ b/cellxgene_gateway/cache_entry.py @@ -6,18 +6,26 @@ # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # OR CONDITIONS OF ANY KIND, either express or implied. See the License for # the specific language governing permissions and limitations under the License. -import psutil -import logging import datetime +import logging -from flask import make_response, request, render_template +import psutil +from enum import Enum +from flask import make_response, render_template, request from requests import get, post, put import re from cellxgene_gateway import env from cellxgene_gateway.cellxgene_exception import CellxgeneException -from cellxgene_gateway.util import current_time_stamp from cellxgene_gateway.flask_util import querystring +from cellxgene_gateway.util import current_time_stamp + + +class CacheEntryStatus(Enum): + loaded = "loaded" + loading = "loading" + error = "error" + terminated = "terminated" class CacheEntry: @@ -28,7 +36,7 @@ class CacheEntry: port, launchtime, timestamp, - status, + status: CacheEntryStatus, message, all_output, stderr, @@ -54,7 +62,7 @@ class CacheEntry: port, current_time_stamp(), current_time_stamp(), - "loading", + CacheEntryStatus.loading, None, None, None, @@ -63,13 +71,13 @@ class CacheEntry: def set_loaded(self, pid): self.pid = pid - self.status = "loaded" + self.status = CacheEntryStatus.loaded def set_error(self, message, stderr, http_status): self.message = message self.stderr = stderr self.http_status = http_status - self.status = "error" + self.status = CacheEntryStatus.error def append_output(self, output): if self.all_output == None: @@ -79,7 +87,7 @@ class CacheEntry: def terminate(self): pid = self.pid - if pid != None and self.status != "terminated": + if pid != None and self.status != CacheEntryStatus.terminated: terminated = [] def on_terminate(p): @@ -96,7 +104,7 @@ class CacheEntry: logging.getLogger("cellxgene_gateway").info( f"terminated {terminated}" ) - self.status = "terminated" + self.status = CacheEntryStatus.terminated def rewrite_text_content(self, cellxgene_content): # for v0.16.0 compatibility, see issue #24 @@ -125,7 +133,7 @@ class CacheEntry: r = make_response(f"Redirect to {gateway_basepath}\n", 301) r.headers["location"] = gateway_basepath + querystring() return r - elif self.status == "loading": + elif self.status == CacheEntryStatus.loading: launch_time = datetime.datetime.fromtimestamp(self.launchtime) return render_template( "loading.html", diff --git a/cellxgene_gateway/cache_key.py b/cellxgene_gateway/cache_key.py index 7b76a36..f45bb92 100644 --- a/cellxgene_gateway/cache_key.py +++ b/cellxgene_gateway/cache_key.py @@ -10,8 +10,8 @@ # There are three kinds of CacheKey: # 1) somedir/dataset.h5ad: a dataset # in this case, pathpart == dataset == 'somedir/dataset.h5ad' -# 2) somedir/dataset_annotations/saldaal1-T5HMVBNV.csv : an actual annotaitons file. -# in this case, pathpart == 'dataset_annotations/saldaal1-T5HMVBNV.csv', dataset == 'somedir/dataset.h5ad' +# 2) somedir/dataset_annotations/my_annotations.csv : an actual annotaitons file. +# in this case, pathpart == 'dataset_annotations/my_annotations.csv', dataset == 'somedir/dataset.h5ad' # 3) somedir/dataset_annotations: an annotation directory. The corresponding h5ad must exist, but the directory may not. # in this case, pathpart == 'dataset_annotations', dataset == 'somedir/dataset.h5ad' diff --git a/cellxgene_gateway/env.py b/cellxgene_gateway/env.py index 0666bd0..bd14ebd 100644 --- a/cellxgene_gateway/env.py +++ b/cellxgene_gateway/env.py @@ -7,8 +7,8 @@ # OR CONDITIONS OF ANY KIND, either express or implied. See the License for # the specific language governing permissions and limitations under the License. -import os import logging +import os import socket cellxgene_location = os.environ.get("CELLXGENE_LOCATION") @@ -31,10 +31,10 @@ enable_upload = os.environ.get("GATEWAY_ENABLE_UPLOAD", "").lower() in [ ] enable_annotations = os.environ.get( "GATEWAY_ENABLE_ANNOTATIONS", "" -).lower() in ["true", "1"] +).lower() in ["true", "1",] enable_backed_mode = os.environ.get( "GATEWAY_ENABLE_BACKED_MODE", "" -).lower() in ["true", "1"] +).lower() in ["true", "1",] env_vars = { "CELLXGENE_LOCATION": cellxgene_location, diff --git a/cellxgene_gateway/extra_scripts.py b/cellxgene_gateway/extra_scripts.py index 7ef49e5..abf2218 100644 --- a/cellxgene_gateway/extra_scripts.py +++ b/cellxgene_gateway/extra_scripts.py @@ -7,9 +7,10 @@ # OR CONDITIONS OF ANY KIND, either express or implied. See the License for # the specific language governing permissions and limitations under the License. -from cellxgene_gateway import env from json import loads +from cellxgene_gateway import env + def get_extra_scripts(): # can be array of script tags to inject on every page, e.g. for google analytics could be diff --git a/cellxgene_gateway/gateway.py b/cellxgene_gateway/gateway.py index b0ff9cf..723d488 100644 --- a/cellxgene_gateway/gateway.py +++ b/cellxgene_gateway/gateway.py @@ -7,16 +7,17 @@ # OR CONDITIONS OF ANY KIND, either express or implied. See the License for # the specific language governing permissions and limitations under the License. +import json +import logging + # import BaseHTTPServer import os -import logging -from threading import Thread, Lock -import json +from threading import Lock, Thread from flask import ( Flask, - redirect, make_response, + redirect, render_template, request, send_from_directory, @@ -27,14 +28,15 @@ from werkzeug.utils import secure_filename from cellxgene_gateway import env from cellxgene_gateway.backend_cache import BackendCache +from cellxgene_gateway.cache_entry import CacheEntryStatus from cellxgene_gateway.cellxgene_exception import CellxgeneException from cellxgene_gateway.dir_util import create_dir, is_subdir -from cellxgene_gateway.filecrawl import recurse_dir, render_entries from cellxgene_gateway.extra_scripts import get_extra_scripts +from cellxgene_gateway.filecrawl import recurse_dir, render_entries +from cellxgene_gateway.path_util import get_key from cellxgene_gateway.process_exception import ProcessException from cellxgene_gateway.prune_process_cache import PruneProcessCache from cellxgene_gateway.util import current_time_stamp -from cellxgene_gateway.path_util import get_key app = Flask(__name__) @@ -238,9 +240,12 @@ def do_view(path): match.timestamp = current_time_stamp() - if match.status == "loaded" or match.status == "loading": + if ( + match.status == CacheEntryStatus.loaded + or match.status == CacheEntryStatus.loading + ): return match.serve_content(path) - elif match.status == "error": + elif match.status == CacheEntryStatus.error: raise ProcessException.from_cache_entry(match) diff --git a/cellxgene_gateway/path_util.py b/cellxgene_gateway/path_util.py index f4a133f..1b23bc8 100644 --- a/cellxgene_gateway/path_util.py +++ b/cellxgene_gateway/path_util.py @@ -12,9 +12,9 @@ import os from flask_api import status from cellxgene_gateway import env +from cellxgene_gateway.cache_key import CacheKey from cellxgene_gateway.cellxgene_exception import CellxgeneException from cellxgene_gateway.dir_util import make_h5ad -from cellxgene_gateway.cache_key import CacheKey def get_key(path): @@ -31,7 +31,7 @@ def get_key(path): return CacheKey(trimmed, trimmed, None) elif trimmed.endswith(".csv"): - # 2) somedir/dataset_annotations/saldaal1-T5HMVBNV.csv : an actual annotations file. + # 2) somedir/dataset_annotations/my_annotations.csv : an actual annotations file. annotations_dir = os.path.split(trimmed)[0] dataset = make_h5ad(annotations_dir) if data_file_exists(dataset): diff --git a/cellxgene_gateway/prune_process_cache.py b/cellxgene_gateway/prune_process_cache.py index f257c10..f332ffd 100644 --- a/cellxgene_gateway/prune_process_cache.py +++ b/cellxgene_gateway/prune_process_cache.py @@ -7,11 +7,11 @@ # OR CONDITIONS OF ANY KIND, either express or implied. See the License for # the specific language governing permissions and limitations under the License. -import time import logging +import time -from cellxgene_gateway.util import current_time_stamp from cellxgene_gateway.env import ttl +from cellxgene_gateway.util import current_time_stamp class PruneProcessCache: diff --git a/cellxgene_gateway/subprocess_backend.py b/cellxgene_gateway/subprocess_backend.py index 628a993..dffdf9c 100644 --- a/cellxgene_gateway/subprocess_backend.py +++ b/cellxgene_gateway/subprocess_backend.py @@ -11,14 +11,15 @@ import logging import subprocess from flask_api import status +from cellxgene_gateway.cache_entry import CacheEntryStatus +from cellxgene_gateway.dir_util import make_annotations +from cellxgene_gateway.path_util import get_annotation_file_path, get_file_path from cellxgene_gateway.env import ( enable_annotations, enable_backed_mode, cellxgene_args, ) from cellxgene_gateway.process_exception import ProcessException -from cellxgene_gateway.dir_util import make_annotations -from cellxgene_gateway.path_util import get_file_path, get_annotation_file_path class SubprocessBackend: @@ -85,7 +86,7 @@ class SubprocessBackend: message = "Cellxgene failed to launch dataset." http_status = status.HTTP_500_INTERNAL_SERVER_ERROR - cache_entry.status = "error" + cache_entry.status = CacheEntryStatus.error cache_entry.set_error(message, stderr, http_status) raise ProcessException.from_cache_entry(cache_entry) diff --git a/cellxgene_gateway/templates/cache_status.html b/cellxgene_gateway/templates/cache_status.html index 34b6f52..956fe56 100644 --- a/cellxgene_gateway/templates/cache_status.html +++ b/cellxgene_gateway/templates/cache_status.html @@ -48,11 +48,11 @@