", methods=["GET"])
def do_terminate(path):
@@ -251,7 +289,10 @@ def do_terminate(path):
def main():
- logging.basicConfig(level=logging.INFO, format='%(asctime)s:%(name)s:%(levelname)s:%(message)s')
+ logging.basicConfig(
+ level=logging.INFO,
+ format="%(asctime)s:%(name)s:%(levelname)s:%(message)s",
+ )
env.validate()
pruner = PruneProcessCache(cache)
diff --git a/cellxgene_gateway/path_util.py b/cellxgene_gateway/path_util.py
index 08e5ccf..1b23bc8 100644
--- a/cellxgene_gateway/path_util.py
+++ b/cellxgene_gateway/path_util.py
@@ -12,9 +12,10 @@ import os
from flask_api import status
from cellxgene_gateway import env
+from cellxgene_gateway.cache_key import CacheKey
from cellxgene_gateway.cellxgene_exception import CellxgeneException
from cellxgene_gateway.dir_util import make_h5ad
-from cellxgene_gateway.cache_key import CacheKey
+
def get_key(path):
if path == "/" or path == "":
@@ -25,33 +26,35 @@ def get_key(path):
trimmed = path[:-1] if path[-1] == "/" else path
try:
# valid paths come in three forms:
- if trimmed.endswith('.h5ad') and data_file_exists(trimmed):
+ if trimmed.endswith(".h5ad") and data_file_exists(trimmed):
# 1) somedir/dataset.h5ad: a dataset
return CacheKey(trimmed, trimmed, None)
- elif trimmed.endswith('.csv'):
+ elif trimmed.endswith(".csv"):
- # 2) somedir/dataset_annotations/saldaal1-T5HMVBNV.csv : an actual annotations file.
+ # 2) somedir/dataset_annotations/my_annotations.csv : an actual annotations file.
annotations_dir = os.path.split(trimmed)[0]
dataset = make_h5ad(annotations_dir)
if data_file_exists(dataset):
data_dir_ensure(annotations_dir)
return CacheKey(trimmed, dataset, trimmed)
- elif trimmed.endswith('_annotations') and data_dir_exists(trimmed):
+ elif trimmed.endswith("_annotations") and data_dir_exists(trimmed):
# 3) somedir/dataset_annotations: an annotation directory. The corresponding h5ad must exist, but the directory may not.
dataset = make_h5ad(trimmed)
if data_file_exists(dataset):
- return CacheKey(trimmed, dataset, '')
+ return CacheKey(trimmed, dataset, "")
except CellxgeneException:
pass
split = os.path.split(trimmed)
return get_key(split[0])
+
def validate_exists(file_path):
if not os.path.exists(file_path):
raise CellxgeneException(
"File does not exist: " + file_path, status.HTTP_400_BAD_REQUEST
)
+
def validate_is_file(file_path):
validate_exists(file_path)
if not os.path.isfile(file_path):
@@ -59,6 +62,8 @@ def validate_is_file(file_path):
"Path is not file: " + file_path, status.HTTP_400_BAD_REQUEST
)
return
+
+
def validate_is_dir(file_path):
validate_exists(file_path)
if not os.path.isdir(file_path):
@@ -67,29 +72,36 @@ def validate_is_dir(file_path):
)
return
+
def data_file_exists(dataset):
file_path = os.path.join(env.cellxgene_data, dataset)
validate_is_file(file_path)
return True
+
+
def data_dir_exists(dataset):
file_path = os.path.join(env.cellxgene_data, dataset)
validate_is_dir(file_path)
return True
+
+
def data_dir_ensure(dataset):
file_path = os.path.join(env.cellxgene_data, dataset)
if not os.path.exists(file_path):
os.makedirs(file_path)
+
def get_file_path(key):
dataset = key.dataset
file_path = os.path.join(env.cellxgene_data, dataset)
validate_is_file(file_path)
return file_path
+
def get_annotation_file_path(key):
if key.annotation_file is None:
return None
- if key.annotation_file == '':
- return ''
+ if key.annotation_file == "":
+ return ""
file_path = os.path.join(env.cellxgene_data, key.annotation_file)
return file_path
diff --git a/cellxgene_gateway/prune_process_cache.py b/cellxgene_gateway/prune_process_cache.py
index 3bb8f2b..f332ffd 100644
--- a/cellxgene_gateway/prune_process_cache.py
+++ b/cellxgene_gateway/prune_process_cache.py
@@ -7,17 +7,17 @@
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
# the specific language governing permissions and limitations under the License.
-import time
import logging
+import time
-from cellxgene_gateway.util import current_time_stamp
from cellxgene_gateway.env import ttl
+from cellxgene_gateway.util import current_time_stamp
class PruneProcessCache:
def __init__(self, cache):
self.cache = cache
- self.expire_seconds = (3600 if ttl is None else int(ttl))
+ self.expire_seconds = 3600 if ttl is None else int(ttl)
def __call__(self):
while True:
@@ -27,14 +27,24 @@ class PruneProcessCache:
def prune(self):
timestamp = current_time_stamp()
cutoff = timestamp - self.expire_seconds
- processes_to_delete = [p for p in self.cache.entry_list if p.timestamp < cutoff]
- processes_to_keep = [p for p in self.cache.entry_list if not p.timestamp < cutoff]
+ processes_to_delete = [
+ p for p in self.cache.entry_list if p.timestamp < cutoff
+ ]
+ processes_to_keep = [
+ p for p in self.cache.entry_list if not p.timestamp < cutoff
+ ]
logger = logging.getLogger("cellxgene_gateway")
- logger.debug(f"Cutoff {cutoff} = timestamp {timestamp} - expire seconds {self.expire_seconds} , keeping {processes_to_keep}")
-
+ logger.debug(
+ f"Cutoff {cutoff} = timestamp {timestamp} - expire seconds {self.expire_seconds} , keeping {processes_to_keep}"
+ )
+
for process in processes_to_delete:
try:
- logger.info(f"pruning process {process.pid} ({process.key.dataset})")
+ logger.info(
+ f"pruning process {process.pid} ({process.key.dataset})"
+ )
self.cache.prune(process)
except Exception:
- logger.exception("failed to prune process {process.pid} ({process.dataset})")
+ logger.exception(
+ "failed to prune process {process.pid} ({process.dataset})"
+ )
diff --git a/cellxgene_gateway/subprocess_backend.py b/cellxgene_gateway/subprocess_backend.py
index 072c680..9c793b5 100644
--- a/cellxgene_gateway/subprocess_backend.py
+++ b/cellxgene_gateway/subprocess_backend.py
@@ -11,17 +11,22 @@ import logging
import subprocess
from flask_api import status
-from cellxgene_gateway.env import enable_annotations, enable_backed_mode
-from cellxgene_gateway.process_exception import ProcessException
+
+from cellxgene_gateway.cache_entry import CacheEntryStatus
from cellxgene_gateway.dir_util import make_annotations
-from cellxgene_gateway.path_util import get_file_path, get_annotation_file_path
+from cellxgene_gateway.env import enable_annotations, enable_backed_mode
+from cellxgene_gateway.path_util import get_annotation_file_path, get_file_path
+from cellxgene_gateway.process_exception import ProcessException
+
class SubprocessBackend:
def __init__(self):
pass
- def create_cmd(self, cellxgene_loc, file_path, port, scripts, annotation_file_path):
+ def create_cmd(
+ self, cellxgene_loc, file_path, port, scripts, annotation_file_path
+ ):
if enable_annotations and not annotation_file_path is None:
if annotation_file_path == "":
extra_args = f" --annotations-dir {make_annotations(file_path)}"
@@ -47,7 +52,11 @@ class SubprocessBackend:
def launch(self, cellxgene_loc, scripts, cache_entry):
cmd = self.create_cmd(
- cellxgene_loc, get_file_path(cache_entry.key), cache_entry.port, scripts, get_annotation_file_path(cache_entry.key)
+ cellxgene_loc,
+ get_file_path(cache_entry.key),
+ cache_entry.port,
+ scripts,
+ get_annotation_file_path(cache_entry.key),
)
logging.getLogger("cellxgene_gateway").info(f"launching {cmd}")
process = subprocess.Popen(
@@ -70,7 +79,7 @@ class SubprocessBackend:
message = "Cellxgene failed to launch dataset."
http_status = status.HTTP_500_INTERNAL_SERVER_ERROR
- cache_entry.status = "error"
+ cache_entry.status = CacheEntryStatus.error
cache_entry.set_error(message, stderr, http_status)
raise ProcessException.from_cache_entry(cache_entry)
diff --git a/cellxgene_gateway/templates/cache_status.html b/cellxgene_gateway/templates/cache_status.html
index 34b6f52..956fe56 100644
--- a/cellxgene_gateway/templates/cache_status.html
+++ b/cellxgene_gateway/templates/cache_status.html
@@ -48,11 +48,11 @@
| {{ entry.port }} |
{{ entry.launchtime }} |
{{ entry.timestamp }} |
- {{ entry.status }} |
+ {{ entry.status.name }} |
{{ entry.message }} |
{{ entry.http_status }} |
- {% if entry.status == 'loaded' %}
+ {% if entry.status.name == 'loaded' %}
terminate
{% endif %}
|
diff --git a/environment.yml b/environment.yml
index 4063d64..912f142 100644
--- a/environment.yml
+++ b/environment.yml
@@ -1,4 +1,4 @@
-name: cellxgene-dev
+name: cellxgene-gateway
channels:
- conda-forge
dependencies:
diff --git a/setup.py b/setup.py
index 9a8dfb8..cfb38d4 100644
--- a/setup.py
+++ b/setup.py
@@ -3,22 +3,25 @@ import codecs
from setuptools import find_packages, setup
import sys
-if sys.version_info < (3,6):
- sys.exit('Sorry, Python < 3.6 is not supported')
+if sys.version_info < (3, 6):
+ sys.exit("Sorry, Python < 3.6 is not supported")
+
def read(rel_path):
here = os.path.abspath(os.path.dirname(__file__))
- with codecs.open(os.path.join(here, rel_path), 'r') as fp:
+ with codecs.open(os.path.join(here, rel_path), "r") as fp:
return fp.read()
+
def get_version(rel_path):
for line in read(rel_path).splitlines():
- if line.startswith('__version__'):
+ if line.startswith("__version__"):
delim = '"' if '"' in line else "'"
return line.split(delim)[1]
else:
raise RuntimeError("Unable to find version string.")
+
def parse_requirements():
reqs = []
with open("requirements.txt", "r") as f:
@@ -26,6 +29,7 @@ def parse_requirements():
reqs.append(l.strip("\n"))
return reqs
+
with open("README.md", "r") as fh:
long_description = fh.read()
@@ -50,13 +54,14 @@ setup(
"cellxgene_gateway": [
"static/css/homepagestyle.css",
"static/nibr.ico",
- "templates/*.html"
- ]},
- data_files=[('', ['README.md', 'LICENSE'])],
+ "templates/*.html",
+ ]
+ },
+ data_files=[("", ["README.md", "LICENSE"])],
install_requires=install_reqs,
entry_points={
"console_scripts": ["cellxgene-gateway=cellxgene_gateway.gateway:main"]
},
classifiers=["Topic :: Scientific/Engineering :: Visualization"],
- python_requires='>=3.6',
+ python_requires=">=3.6",
)
diff --git a/tests/test_cache_entry.py b/tests/test_cache_entry.py
new file mode 100644
index 0000000..b3a3c4b
--- /dev/null
+++ b/tests/test_cache_entry.py
@@ -0,0 +1,13 @@
+import unittest
+from unittest.mock import MagicMock, patch
+
+from cellxgene_gateway.cache_entry import CacheEntry, CacheEntryStatus
+
+
+class TestCacheEntry(unittest.TestCase):
+ def test_GIVEN_key_and_port_THEN_returns_loading_CacheEntry(self):
+ entry = CacheEntry.for_key("some-key", 1)
+ self.assertEqual(entry.status, CacheEntryStatus.loading)
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_dir_util.py b/tests/test_dir_util.py
deleted file mode 100644
index 3a5e5dc..0000000
--- a/tests/test_dir_util.py
+++ /dev/null
@@ -1,38 +0,0 @@
-import unittest
-from unittest.mock import MagicMock, patch
-from cellxgene_gateway.dir_util import render_entry
-
-class TestRenderEntry(unittest.TestCase):
- def test_GIVEN_path_both_slash_THEN_view_has_single_slash(self):
- entry = {
- "path": "/somepath/",
- "name": "entry",
- "type": "file",
- }
- rendered = render_entry(entry)
- self.assertIn('view/somepath', rendered)
- def test_GIVEN_path_starts_slash_THEN_view_has_single_slash(self):
- entry = {
- "path": "/somepath",
- "name": "entry",
- "type": "file",
- }
- rendered = render_entry(entry)
- self.assertIn('view/somepath', rendered)
- def test_GIVEN_path_ends_slash_THEN_view_has_single_slash(self):
- entry = {
- "path": "somepath/",
- "name": "entry",
- "type": "file",
- }
- rendered = render_entry(entry)
- self.assertIn('view/somepath', rendered)
- def test_GIVEN_path_no_slash_THEN_view_has_single_slash(self):
- entry = {
- "path": "somepath",
- "name": "entry",
- "type": "file",
- }
- rendered = render_entry(entry)
- self.assertIn('view/somepath', rendered)
-
\ No newline at end of file
diff --git a/tests/test_extra_scripts.py b/tests/test_extra_scripts.py
index 4e63c89..c429c55 100644
--- a/tests/test_extra_scripts.py
+++ b/tests/test_extra_scripts.py
@@ -1,23 +1,26 @@
import unittest
from unittest.mock import MagicMock, patch
+
from cellxgene_gateway.extra_scripts import get_extra_scripts
+
class TestExtraScripts(unittest.TestCase):
- @patch('cellxgene_gateway.env.extra_scripts', new='["abc","def"]')
+ @patch("cellxgene_gateway.env.extra_scripts", new='["abc","def"]')
def test_GIVEN_two_scripts_THEN_returns_two_strings(self):
- self.assertEqual(get_extra_scripts(), ['abc', 'def'])
+ self.assertEqual(get_extra_scripts(), ["abc", "def"])
- @patch('cellxgene_gateway.env.extra_scripts', new='["abc", "def"]')
+ @patch("cellxgene_gateway.env.extra_scripts", new='["abc", "def"]')
def test_GIVEN_two_scripts_space_THEN_returns_two_strings(self):
- self.assertEqual(get_extra_scripts(), ['abc', 'def'])
+ self.assertEqual(get_extra_scripts(), ["abc", "def"])
- @patch('cellxgene_gateway.env.extra_scripts', new=None)
+ @patch("cellxgene_gateway.env.extra_scripts", new=None)
def test_GIVEN_none_THEN_returns_empty_array(self):
self.assertEqual(get_extra_scripts(), [])
- @patch('cellxgene_gateway.env.extra_scripts', new='[]')
+ @patch("cellxgene_gateway.env.extra_scripts", new="[]")
def test_GIVEN_empty_string_THEN_returns_empty_array(self):
self.assertEqual(get_extra_scripts(), [])
-if __name__ == '__main__':
- unittest.main()
\ No newline at end of file
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_filecrawl.py b/tests/test_filecrawl.py
new file mode 100644
index 0000000..bf30259
--- /dev/null
+++ b/tests/test_filecrawl.py
@@ -0,0 +1,46 @@
+import unittest
+from unittest.mock import MagicMock, patch
+
+from cellxgene_gateway.filecrawl import render_entry
+
+
+class TestRenderEntry(unittest.TestCase):
+ def test_GIVEN_path_both_slash_THEN_view_has_single_slash(self):
+ entry = {
+ "path": "/somepath/",
+ "name": "entry",
+ "type": "file",
+ "annotations": []
+ }
+ rendered = render_entry(entry)
+ self.assertIn("view/somepath", rendered)
+
+ def test_GIVEN_path_starts_slash_THEN_view_has_single_slash(self):
+ entry = {
+ "path": "/somepath",
+ "name": "entry",
+ "type": "file",
+ "annotations": []
+ }
+ rendered = render_entry(entry)
+ self.assertIn("view/somepath", rendered)
+
+ def test_GIVEN_path_ends_slash_THEN_view_has_single_slash(self):
+ entry = {
+ "path": "somepath/",
+ "name": "entry",
+ "type": "file",
+ "annotations": []
+ }
+ rendered = render_entry(entry)
+ self.assertIn("view/somepath", rendered)
+
+ def test_GIVEN_path_no_slash_THEN_view_has_single_slash(self):
+ entry = {
+ "path": "somepath",
+ "name": "entry",
+ "type": "file",
+ "annotations": []
+ }
+ rendered = render_entry(entry)
+ self.assertIn("view/somepath", rendered)
diff --git a/tests/test_prune_process_cache.py b/tests/test_prune_process_cache.py
index fe84677..3609e3a 100644
--- a/tests/test_prune_process_cache.py
+++ b/tests/test_prune_process_cache.py
@@ -1,13 +1,15 @@
import unittest
from unittest.mock import MagicMock, patch
-from cellxgene_gateway.cache_entry import CacheEntry
+
from cellxgene_gateway.backend_cache import BackendCache
+from cellxgene_gateway.cache_entry import CacheEntry
+
class TestPruneProcessCache(unittest.TestCase):
- @patch('cellxgene_gateway.util.current_time_stamp', new=lambda:0)
- @patch('cellxgene_gateway.env.ttl', new='10')
- @patch('cellxgene_gateway.cache_entry.CacheEntry')
- @patch('cellxgene_gateway.cache_entry.CacheEntry')
+ @patch("cellxgene_gateway.util.current_time_stamp", new=lambda: 0)
+ @patch("cellxgene_gateway.env.ttl", new="10")
+ @patch("cellxgene_gateway.cache_entry.CacheEntry")
+ @patch("cellxgene_gateway.cache_entry.CacheEntry")
def test_GIVEN_one_old_one_new_THEN_prune_old(self, old, new):
from cellxgene_gateway.prune_process_cache import PruneProcessCache
@@ -22,5 +24,6 @@ class TestPruneProcessCache(unittest.TestCase):
self.assertEqual(len(cache.entry_list), 1)
self.assertEqual(cache.entry_list[0], new)
-if __name__ == '__main__':
- unittest.main()
\ No newline at end of file
+
+if __name__ == "__main__":
+ unittest.main()