Merge branch 'gmerge' into gmaster

# Conflicts:
#	cellxgene_gateway/backend_cache.py
#	cellxgene_gateway/cache_entry.py
#	cellxgene_gateway/env.py
#	cellxgene_gateway/gateway.py
#	cellxgene_gateway/subprocess_backend.py
#	tests/test_cache_entry.py
#	tests/test_dir_util.py

cellxgene_gateway/backend_cache.py

@@ -13,7 +13,7 @@ from threading import Thread
 from flask_api import status
 
 from cellxgene_gateway import env
-from cellxgene_gateway.cache_entry import CacheEntry
+from cellxgene_gateway.cache_entry import CacheEntry, CacheEntryStatus
 from cellxgene_gateway.cellxgene_exception import CellxgeneException
 from cellxgene_gateway.subprocess_backend import SubprocessBackend
 
@@ -42,7 +42,7 @@ class BackendCache:
             for c in contents
             if c.key.dataset == key.dataset
             and c.key.annotation_file == key.annotation_file
-            and c.status != "terminated"
+            and c.status != CacheEntryStatus.terminated
         ]
 
         if len(matches) == 0:

cellxgene_gateway/cache_entry.py

@@ -6,18 +6,26 @@
 # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
-import psutil
-import logging
 import datetime
+import logging
 
-from flask import make_response, request, render_template
+import psutil
+from enum import Enum
+from flask import make_response, render_template, request
 from requests import get, post, put
 import re
 
 from cellxgene_gateway import env
 from cellxgene_gateway.cellxgene_exception import CellxgeneException
-from cellxgene_gateway.util import current_time_stamp
 from cellxgene_gateway.flask_util import querystring
+from cellxgene_gateway.util import current_time_stamp
+
+
+class CacheEntryStatus(Enum):
+    loaded = "loaded"
+    loading = "loading"
+    error = "error"
+    terminated = "terminated"
 
 
 class CacheEntry:
@@ -28,7 +36,7 @@ class CacheEntry:
         port,
         launchtime,
         timestamp,
-        status,
+        status: CacheEntryStatus,
         message,
         all_output,
         stderr,
@@ -54,7 +62,7 @@ class CacheEntry:
             port,
             current_time_stamp(),
             current_time_stamp(),
-            "loading",
+            CacheEntryStatus.loading,
             None,
             None,
             None,
@@ -63,13 +71,13 @@ class CacheEntry:
 
     def set_loaded(self, pid):
         self.pid = pid
-        self.status = "loaded"
+        self.status = CacheEntryStatus.loaded
 
     def set_error(self, message, stderr, http_status):
         self.message = message
         self.stderr = stderr
         self.http_status = http_status
-        self.status = "error"
+        self.status = CacheEntryStatus.error
 
     def append_output(self, output):
         if self.all_output == None:
@@ -79,7 +87,7 @@ class CacheEntry:
 
     def terminate(self):
         pid = self.pid
-        if pid != None and self.status != "terminated":
+        if pid != None and self.status != CacheEntryStatus.terminated:
             terminated = []
 
             def on_terminate(p):
@@ -96,7 +104,7 @@ class CacheEntry:
             logging.getLogger("cellxgene_gateway").info(
                 f"terminated {terminated}"
             )
-        self.status = "terminated"
+        self.status = CacheEntryStatus.terminated
 
     def rewrite_text_content(self, cellxgene_content):
         # for v0.16.0 compatibility, see issue #24
@@ -125,7 +133,7 @@ class CacheEntry:
             r = make_response(f"Redirect to {gateway_basepath}\n", 301)
             r.headers["location"] = gateway_basepath + querystring()
             return r
-        elif self.status == "loading":
+        elif self.status == CacheEntryStatus.loading:
             launch_time = datetime.datetime.fromtimestamp(self.launchtime)
             return render_template(
                 "loading.html",

cellxgene_gateway/cache_key.py

@@ -10,8 +10,8 @@
 # There are three kinds of CacheKey:
 # 1) somedir/dataset.h5ad: a dataset
 #    in this case, pathpart == dataset == 'somedir/dataset.h5ad'
-# 2) somedir/dataset_annotations/saldaal1-T5HMVBNV.csv : an actual annotaitons file.
-#    in this case, pathpart == 'dataset_annotations/saldaal1-T5HMVBNV.csv', dataset == 'somedir/dataset.h5ad'
+# 2) somedir/dataset_annotations/my_annotations.csv : an actual annotaitons file.
+#    in this case, pathpart == 'dataset_annotations/my_annotations.csv', dataset == 'somedir/dataset.h5ad'
 # 3) somedir/dataset_annotations: an annotation directory. The corresponding h5ad must exist, but the directory may not.
 #    in this case, pathpart == 'dataset_annotations', dataset == 'somedir/dataset.h5ad'
 

cellxgene_gateway/env.py

@@ -7,8 +7,8 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
-import os
 import logging
+import os
 import socket
 
 cellxgene_location = os.environ.get("CELLXGENE_LOCATION")
@@ -31,10 +31,10 @@ enable_upload = os.environ.get("GATEWAY_ENABLE_UPLOAD", "").lower() in [
 ]
 enable_annotations = os.environ.get(
     "GATEWAY_ENABLE_ANNOTATIONS", ""
-).lower() in ["true", "1"]
+).lower() in ["true", "1",]
 enable_backed_mode = os.environ.get(
     "GATEWAY_ENABLE_BACKED_MODE", ""
-).lower() in ["true", "1"]
+).lower() in ["true", "1",]
 
 env_vars = {
     "CELLXGENE_LOCATION": cellxgene_location,

cellxgene_gateway/extra_scripts.py

@@ -7,9 +7,10 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
-from cellxgene_gateway import env
 from json import loads
 
+from cellxgene_gateway import env
+
 
 def get_extra_scripts():
     # can be array of script tags to inject on every page, e.g. for google analytics could be

cellxgene_gateway/gateway.py

@@ -7,16 +7,17 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
+import json
+import logging
+
 # import BaseHTTPServer
 import os
-import logging
-from threading import Thread, Lock
-import json
+from threading import Lock, Thread
 
 from flask import (
     Flask,
-    redirect,
     make_response,
+    redirect,
     render_template,
     request,
     send_from_directory,
@@ -27,14 +28,15 @@ from werkzeug.utils import secure_filename
 
 from cellxgene_gateway import env
 from cellxgene_gateway.backend_cache import BackendCache
+from cellxgene_gateway.cache_entry import CacheEntryStatus
 from cellxgene_gateway.cellxgene_exception import CellxgeneException
 from cellxgene_gateway.dir_util import create_dir, is_subdir
-from cellxgene_gateway.filecrawl import recurse_dir, render_entries
 from cellxgene_gateway.extra_scripts import get_extra_scripts
+from cellxgene_gateway.filecrawl import recurse_dir, render_entries
+from cellxgene_gateway.path_util import get_key
 from cellxgene_gateway.process_exception import ProcessException
 from cellxgene_gateway.prune_process_cache import PruneProcessCache
 from cellxgene_gateway.util import current_time_stamp
-from cellxgene_gateway.path_util import get_key
 
 app = Flask(__name__)
 
@@ -238,9 +240,12 @@ def do_view(path):
 
     match.timestamp = current_time_stamp()
 
-    if match.status == "loaded" or match.status == "loading":
+    if (
+        match.status == CacheEntryStatus.loaded
+        or match.status == CacheEntryStatus.loading
+    ):
         return match.serve_content(path)
-    elif match.status == "error":
+    elif match.status == CacheEntryStatus.error:
         raise ProcessException.from_cache_entry(match)
 
 

cellxgene_gateway/path_util.py

@@ -12,9 +12,9 @@ import os
 from flask_api import status
 
 from cellxgene_gateway import env
+from cellxgene_gateway.cache_key import CacheKey
 from cellxgene_gateway.cellxgene_exception import CellxgeneException
 from cellxgene_gateway.dir_util import make_h5ad
-from cellxgene_gateway.cache_key import CacheKey
 
 
 def get_key(path):
@@ -31,7 +31,7 @@ def get_key(path):
             return CacheKey(trimmed, trimmed, None)
         elif trimmed.endswith(".csv"):
 
-            # 2) somedir/dataset_annotations/saldaal1-T5HMVBNV.csv : an actual annotations file.
+            # 2) somedir/dataset_annotations/my_annotations.csv : an actual annotations file.
             annotations_dir = os.path.split(trimmed)[0]
             dataset = make_h5ad(annotations_dir)
             if data_file_exists(dataset):

cellxgene_gateway/prune_process_cache.py

@@ -7,11 +7,11 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
-import time
 import logging
+import time
 
-from cellxgene_gateway.util import current_time_stamp
 from cellxgene_gateway.env import ttl
+from cellxgene_gateway.util import current_time_stamp
 
 
 class PruneProcessCache:

cellxgene_gateway/subprocess_backend.py

@@ -11,14 +11,15 @@ import logging
 import subprocess
 
 from flask_api import status
+from cellxgene_gateway.cache_entry import CacheEntryStatus
+from cellxgene_gateway.dir_util import make_annotations
+from cellxgene_gateway.path_util import get_annotation_file_path, get_file_path
 from cellxgene_gateway.env import (
     enable_annotations,
     enable_backed_mode,
     cellxgene_args,
 )
 from cellxgene_gateway.process_exception import ProcessException
-from cellxgene_gateway.dir_util import make_annotations
-from cellxgene_gateway.path_util import get_file_path, get_annotation_file_path
 
 
 class SubprocessBackend:
@@ -85,7 +86,7 @@ class SubprocessBackend:
                     message = "Cellxgene failed to launch dataset."
                     http_status = status.HTTP_500_INTERNAL_SERVER_ERROR
 
-                cache_entry.status = "error"
+                cache_entry.status = CacheEntryStatus.error
                 cache_entry.set_error(message, stderr, http_status)
 
                 raise ProcessException.from_cache_entry(cache_entry)

cellxgene_gateway/templates/cache_status.html

@@ -48,11 +48,11 @@
                <td>{{ entry.port }}</td>
                <td class="timestamp">{{ entry.launchtime }}</td>
                <td class="timestamp">{{ entry.timestamp }}</td>
-               <td>{{ entry.status }}</td>
+               <td>{{ entry.status.name }}</td>
                <td>{{ entry.message }}</td>
                <td>{{ entry.http_status }}</td>
                <td>
-                 {% if entry.status == 'loaded' %}
+                 {% if entry.status.name == 'loaded' %}
                    <a href="{{ url_for('do_terminate', path=entry.key.pathpart) }}"> terminate </a>
                  {% endif %}
                </td>

environment.yml

@@ -1,4 +1,4 @@
-name: cellxgene-dev
+name: cellxgene-gateway
 channels:
 - conda-forge
 dependencies:

tests/test_cache_entry.py

@@ -1,11 +1,15 @@
 import unittest
-from cellxgene_gateway.cache_entry import CacheEntry
+from cellxgene_gateway.cache_entry import CacheEntry, CacheEntryStatus
 from cellxgene_gateway.cache_key import CacheKey
 
 key = CacheKey("czi/pbmc3k.h5ad", "pbmc3k.h5ad", "tmp.csv")
 
 
 class TestRenderEntry(unittest.TestCase):
+    def test_GIVEN_key_and_port_THEN_returns_loading_CacheEntry(self):
+        entry = CacheEntry.for_key("some-key", 1)
+        self.assertEqual(entry.status, CacheEntryStatus.loading)
+
     def test_GIVEN_absolute_static_url_THEN_include_path(self):
         actual = CacheEntry.for_key(key, 8000).rewrite_text_content(
             "src:url(/static/assets/"
@@ -21,3 +25,7 @@ class TestRenderEntry(unittest.TestCase):
         )
         expected = '<link rel="shortcut icon" href="http://localhost:5005/view/czi/pbmc3k.h5ad/static/assets/favicon.ico">'
         self.assertEqual(actual, expected)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_extra_scripts.py

@@ -1,5 +1,6 @@
 import unittest
 from unittest.mock import MagicMock, patch
+
 from cellxgene_gateway.extra_scripts import get_extra_scripts
 
 

tests/test_filecrawl.py

@@ -0,0 +1,46 @@
+import unittest
+from unittest.mock import MagicMock, patch
+
+from cellxgene_gateway.filecrawl import render_entry
+
+
+class TestRenderEntry(unittest.TestCase):
+    def test_GIVEN_path_both_slash_THEN_view_has_single_slash(self):
+        entry = {
+            "path": "/somepath/",
+            "name": "entry",
+            "type": "file",
+            "annotations": [],
+        }
+        rendered = render_entry(entry)
+        self.assertIn("view/somepath", rendered)
+
+    def test_GIVEN_path_starts_slash_THEN_view_has_single_slash(self):
+        entry = {
+            "path": "/somepath",
+            "name": "entry",
+            "type": "file",
+            "annotations": [],
+        }
+        rendered = render_entry(entry)
+        self.assertIn("view/somepath", rendered)
+
+    def test_GIVEN_path_ends_slash_THEN_view_has_single_slash(self):
+        entry = {
+            "path": "somepath/",
+            "name": "entry",
+            "type": "file",
+            "annotations": [],
+        }
+        rendered = render_entry(entry)
+        self.assertIn("view/somepath", rendered)
+
+    def test_GIVEN_path_no_slash_THEN_view_has_single_slash(self):
+        entry = {
+            "path": "somepath",
+            "name": "entry",
+            "type": "file",
+            "annotations": [],
+        }
+        rendered = render_entry(entry)
+        self.assertIn("view/somepath", rendered)

tests/test_prune_process_cache.py

@@ -1,7 +1,8 @@
 import unittest
 from unittest.mock import MagicMock, patch
-from cellxgene_gateway.cache_entry import CacheEntry
+
 from cellxgene_gateway.backend_cache import BackendCache
+from cellxgene_gateway.cache_entry import CacheEntry
 
 
 class TestPruneProcessCache(unittest.TestCase):