#9 only prune entries that have a pid

.github/workflows/pr-checks.yaml

@@ -1,49 +0,0 @@
-# Tests that run on every PR
-
-name: Pull Request Checks
-
-on: [push, pull_request]
-
-jobs:
-  black:
-    runs-on: ubuntu-18.04
-    steps:
-      - uses: actions/checkout@v2
-        name: Checkout repository
-
-      - uses: actions/setup-python@v2
-        name: Setup Python
-        with:
-          python-version: 3.9
-
-      - name: Install black
-        run: |
-          python -m pip install --upgrade pip
-          pip install black
-      - name: Run black
-        run: |
-          black -l 79 . --check
-  # This job is copied over from `deploy.yaml`
-  run-tests:
-    runs-on: ubuntu-18.04
-    steps:
-      - uses: actions/checkout@v2
-
-      # See: https://github.com/marketplace/actions/setup-conda
-      - uses: s-weigand/setup-conda@v1
-        with:
-          conda-channels: "conda-forge"
-
-      - name: Build environment
-        run: |
-          conda env create -f environment.yml
-          eval "$(conda shell.bash hook)"
-          conda activate cellxgene-gateway
-          python setup.py install
-
-      - name: Run tests
-        run: |
-          eval "$(conda shell.bash hook)"
-          conda activate cellxgene-gateway
-          python -m unittest discover tests
-          bash <(curl -s https://codecov.io/bash)

.gitignore

@@ -136,6 +136,3 @@ dmypy.json
 .pyre/
 
 # End of https://www.gitignore.io/api/python
-
-*.patch
-.vscode

.travis.yml

@@ -1,41 +0,0 @@
-# This is necessary for nxviz as matplotlib is involved.
-# before_script:
-#   - "export DISPLAY=:99.0"
-#   - "sh -e /etc/init.d/xvfb start"
-#   - sleep 5 # give xvfb some time to start
-
-language: python
-matrix:
-  include:
-    - python: 3.5 # we don't actually use this
-      env: PYTHON_VERSION=3.7
-
-install:
-  # We do this conditionally because it saves us some downloading if the
-  # version is the same.
-  - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
-  - bash miniconda.sh -b -p $HOME/miniconda
-  - export PATH="$HOME/miniconda/bin:$PATH"
-  - hash -r
-  - conda config --set always_yes yes --set changeps1 no
-  - conda update -q conda
-  - conda config --add channels conda-forge
-
-  # Useful for debugging any issues with conda
-  - conda info -a
-
-  # Install Python, py.test, and required packages.
-  - conda env create -f environment.yml
-  - source activate cellxgene-gateway
-  - python setup.py install
-
-script:
-  # Your test script goes here
-  - black -l 79 . --check
-  - python -m unittest discover tests
-
-after_success:
-  - bash <(curl -s https://codecov.io/bash)
-
-notifications:
-  email: true

Changelog.md

@@ -1,21 +0,0 @@
-# 0.2.3
-
-* Added support for ProxyFix
-
-# 0.2.2
-
-  * Fixed bug with annotations (missing annotation.js asset)
-
-# 0.2.1
-
-  * Minor fixes to enable cellxgene 0.16.0
-  * Added CELLXGENE_ARGS to enable passing additional arguments to cellxgene
-  * added metadata/ip_address endpoint
-
-# 0.2.0
-
-Incrementing minor version since the changes for 0.15 are breaking, and we may want to release bugfixes from 0.1.0 branch.
-
-# 0.1.1
-
-Added support for cellxgene 0.15

Readme.md

@@ -30,7 +30,7 @@ Note: you may need to downgrade h5py with `pip install h5py==2.9.0` due to an [i
 ### Option 2: Install from PyPI
 
 ```bash
-pip install cellxgene-gateway
+# NOT YET DONE, COMING! STAY TUNED
 ```
 
 ## Running cellxgene gateway
@@ -39,7 +39,7 @@ pip install cellxgene-gateway
 
 ```bash
 mkdir ../cellxgene_data
-wget https://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/example-dataset/pbmc3k.h5ad -O ../cellxgene_data/pbmc3k.h5ad
+wget https://github.com/chanzuckerberg/cellxgene/raw/master/example-dataset/pbmc3k.h5ad -O ../cellxgene_data/pbmc3k.h5ad
 ```
 
 
@@ -48,6 +48,9 @@ wget https://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/example-d
 ```bash
 export CELLXGENE_DATA=../cellxgene_data  # change this directory if you put data in a different place.
 export CELLXGENE_LOCATION=`which cellxgene`
+export GATEWAY_HOST=localhost:5005
+export GATEWAY_PROTOCOL=http
+export GATEWAY_IP=127.0.0.1
 ```
 
 3. Now, execute the cellxgene gateway:
@@ -60,24 +63,12 @@ Here's what the environment variables mean:
 
 * `CELLXGENE_LOCATION` - the location of the cellxgene executable, e.g. `~/anaconda2/envs/cellxgene/bin/cellxgene`
 * `CELLXGENE_DATA` - a directory that can contain subdirectories with `.h5ad` data files, *without* trailing slash, e.g. `/mnt/cellxgene_data`
-
+* `GATEWAY_HOST` - the hostname and port that the gateway will run on, typically `localhost:5005` if running locally
+* `GATEWAY_PROTOCOL` - typically http when running locally, can be https when deployed if the gateway is behind a load balancer or reverse proxy.
+* `GATEWAY_IP` - ip addess of instance gateway is running on, mostly used to display SSH instructions
 Optional environment variables:
-* `CELLXGENE_ARGS` - catch-all variable that can be used to pass additional command line args to cellxgene server
-* `EXTERNAL_HOST` - the hostname and port from the perspective of the web browser, typically `localhost:5005` if running locally. Defaults to "localhost:{GATEWAY_PORT}"
-* `EXTERNAL_PROTOCOL` - typically http when running locally, can be https when deployed if the gateway is behind a load balancer or reverse proxy that performs https termination. Default value "http"
-* `GATEWAY_IP` - ip addess of instance gateway is running on, mostly used to display SSH instructions. Defaults to `socket.gethostbyname(socket.gethostname())`
-* `GATEWAY_PORT` - local port that the gateway should bind to, defaults to 5005
 * `GATEWAY_EXTRA_SCRIPTS` - JSON array of script paths, will be embedded into each page and forwarded with `--scripts` to cellxgene server
 * `GATEWAY_ENABLE_UPLOAD` - Set to `true` or `1` to enable HTTP uploads. This is not recommended for a public server.
-* `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations. 
-* `GATEWAY_ENABLE_BACKED_MODE` - Set to `true` or to `1` to load AnnData in file-backed mode. This saves memory and speeds up launch time but may reduce overall performance.
-
-If any of the following optional variables are set, [ProxyFix](https://werkzeug.palletsprojects.com/en/1.0.x/middleware/proxy_fix/) will be used.
-* `PROXY_FIX_FOR` - Number of upstream proxies setting X-Forwarded-For
-* `PROXY_FIX_PROTO` - Number of upstream proxies setting X-Forwarded-Proto
-* `PROXY_FIX_HOST` - Number of upstream proxies setting X-Forwarded-Host
-* `PROXY_FIX_PORT` - Number of upstream proxies setting X-Forwarded-Port
-* `PROXY_FIX_PREFIX` - Number of upstream proxies setting X-Forwarded-Prefix
 
 The defaults should be fine if you set up a venv and cellxgene_data folder as above.
 
@@ -121,18 +112,8 @@ python setup.py develop
 
 For convenience, the code repo includes a `run.sh.example` shell script to run the gateway.
 
-4. Install pre-commit hooks
-
-```bash
-conda install -c conda-forge pre-commit
-pre-commit install
-```
-
-
 ## Running Tests
 
-[![Build Status](https://travis-ci.org/Novartis/cellxgene-gateway.svg?branch=master)](https://travis-ci.org/Novartis/cellxgene-gateway)
-
 ```bash
     python -m unittest discover tests
 ```

cellxgene_gateway/__init__.py

@@ -6,5 +6,3 @@
 # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
-
-__version__ = "0.2.3"

cellxgene_gateway/backend_cache.py

@@ -13,7 +13,7 @@ from threading import Thread
 from flask_api import status
 
 from cellxgene_gateway import env
-from cellxgene_gateway.cache_entry import CacheEntry, CacheEntryStatus
+from cellxgene_gateway.cache_entry import CacheEntry
 from cellxgene_gateway.cellxgene_exception import CellxgeneException
 from cellxgene_gateway.subprocess_backend import SubprocessBackend
 
@@ -22,10 +22,8 @@ process_backend = SubprocessBackend()
 
 def is_port_in_use(port):
     import socket
-
     with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-        return s.connect_ex(("localhost", port)) == 0
-
+        return s.connect_ex(('localhost', port)) == 0
 
 class BackendCache:
     def __init__(self):
@@ -35,14 +33,12 @@ class BackendCache:
         contents = self.entry_list
         return [c.port for c in contents]
 
-    def check_entry(self, key):
+    def check_entry(self, dataset):
         contents = self.entry_list
         matches = [
             c
             for c in contents
-            if c.key.dataset == key.dataset
-            and c.key.annotation_file == key.annotation_file
-            and c.status != CacheEntryStatus.terminated
+            if c.dataset == dataset and c.status != "terminated"
         ]
 
         if len(matches) == 0:
@@ -55,14 +51,13 @@ class BackendCache:
                 "Found " + str(len(matches)) + " for " + dataset,
             )
 
-    def create_entry(self, key, scripts):
+    def create_entry(self, dataset, file_path, scripts):
         port = 8000
         existing_ports = self.get_ports()
-
         while (port in existing_ports) or is_port_in_use(port):
             port += 1
 
-        entry = CacheEntry.for_key(key, port)
+        entry = CacheEntry.for_dataset(dataset, file_path, port)
 
         background_thread = Thread(
             target=process_backend.launch,

cellxgene_gateway/cache_entry.py

@@ -6,46 +6,34 @@
 # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
-import datetime
-import logging
-from flask.helpers import url_for
-from flask.wrappers import Response
-
 import psutil
-from enum import Enum
-from flask import make_response, render_template, request
+import logging
+
+from flask import make_response, request
 from requests import get, post, put
-import re
 
 from cellxgene_gateway import env
 from cellxgene_gateway.cellxgene_exception import CellxgeneException
-from cellxgene_gateway.flask_util import querystring
 from cellxgene_gateway.util import current_time_stamp
 
-
-class CacheEntryStatus(Enum):
-    loaded = "loaded"
-    loading = "loading"
-    error = "error"
-    terminated = "terminated"
-
-
 class CacheEntry:
     def __init__(
         self,
         pid,
-        key,
+        dataset,
+        file_path,
         port,
         launchtime,
         timestamp,
-        status: CacheEntryStatus,
+        status,
         message,
         all_output,
         stderr,
         http_status,
     ):
         self.pid = pid
-        self.key = key
+        self.dataset = dataset
+        self.file_path = file_path
         self.port = port
         self.launchtime = launchtime
         self.timestamp = timestamp
@@ -56,15 +44,15 @@ class CacheEntry:
         self.http_status = http_status
 
     @classmethod
-    def for_key(cls, key, port):
-
+    def for_dataset(cls, dataset, file_path, port):
         return cls(
             None,
-            key,
+            dataset,
+            file_path,
             port,
             current_time_stamp(),
             current_time_stamp(),
-            CacheEntryStatus.loading,
+            "loading",
             None,
             None,
             None,
@@ -73,13 +61,13 @@ class CacheEntry:
 
     def set_loaded(self, pid):
         self.pid = pid
-        self.status = CacheEntryStatus.loaded
+        self.status = "loaded"
 
     def set_error(self, message, stderr, http_status):
         self.message = message
         self.stderr = stderr
         self.http_status = http_status
-        self.status = CacheEntryStatus.error
+        self.status = "error"
 
     def append_output(self, output):
         if self.all_output == None:
@@ -89,12 +77,10 @@ class CacheEntry:
 
     def terminate(self):
         pid = self.pid
-        if pid != None and self.status != CacheEntryStatus.terminated:
+        if pid != None and self.status != "terminated":
             terminated = []
-
             def on_terminate(p):
                 terminated.append(p.pid)
-
             p = psutil.Process(pid)
             children = p.children()
             for child in children:
@@ -103,83 +89,49 @@ class CacheEntry:
             terminated.append(p.pid)
             p.terminate()
             psutil.wait_procs([p], callback=on_terminate)
-            logging.getLogger("cellxgene_gateway").info(
-                f"terminated {terminated}"
-            )
-        self.status = CacheEntryStatus.terminated
-
-    def rewrite_text_content(self, cellxgene_content):
-        # for v0.16.0 compatibility, see issue #24
-        gateway_content = (
-            re.sub(
-                '(="|\()/static/',
-                f"\\1{self.gateway_basepath()}static/",
-                cellxgene_content,
-            )
-            .replace("http://fonts.gstatic.com", "https://fonts.gstatic.com")
-            .replace(self.cellxgene_basepath(), self.gateway_basepath())
-        )
-        return gateway_content
-
-    def gateway_basepath(self):
-        return url_for("do_view", path=self.key.pathpart) + "/"
-
-    def cellxgene_basepath(self):
-        return f"http://127.0.0.1:{self.port}"
+            logging.getLogger("cellxgene_gateway").info(f"terminated {terminated}")
+        self.status = "terminated"
 
     def serve_content(self, path):
-        gateway_basepath = self.gateway_basepath()
-        subpath = path[len(self.key.pathpart) :]  # noqa: E203
+        dataset = self.dataset
+
+        gateway_basepath = (
+            f"{env.gateway_protocol}://{env.gateway_host}/view/{dataset}/"
+        )
+        subpath = path[len(dataset) :]  # noqa: E203
 
         if len(subpath) == 0:
-            r = make_response(f"Redirect to {gateway_basepath}\n", 302)
-            r.headers["location"] = gateway_basepath + querystring()
+            r = make_response(f"Redirect to {gateway_basepath}\n", 301)
+            r.headers["location"] = gateway_basepath
             return r
-        elif self.status == CacheEntryStatus.loading:
-            launch_time = datetime.datetime.fromtimestamp(self.launchtime)
-            return render_template(
-                "loading.html",
-                launchtime=launch_time,
-                all_output=self.all_output,
-            )
+
+        port = self.port
+        cellxgene_basepath = f"http://127.0.0.1:{port}"
 
         headers = {}
-        copy_headers = [
-            "accept",
-            "accept-encoding",
-            "accept-language",
-            "cache-control",
-            "connection",
-            "content-length",
-            "content-type",
-            "cookie",
-            "host",
-            "origin",
-            "pragma",
-            "referer",
-            "sec-fetch-mode",
-            "sec-fetch-site",
-            "user-agent",
-        ]
-        for h in copy_headers:
-            if h in request.headers:
-                headers[h] = request.headers[h]
 
-        full_path = self.cellxgene_basepath() + subpath + querystring()
+        if "accept" in request.headers:
+            headers["accept"] = request.headers["accept"]
+        if "user-agent" in request.headers:
+            headers["user-agent"] = request.headers["user-agent"]
+        if "content-type" in request.headers:
+            headers["content-type"] = request.headers["content-type"]
 
         if request.method in ["GET", "HEAD", "OPTIONS"]:
-            cellxgene_response = get(full_path, headers=headers)
+            cellxgene_response = get(
+                cellxgene_basepath + subpath, headers=headers
+            )
         elif request.method == "PUT":
             cellxgene_response = put(
-                full_path,
+                cellxgene_basepath + subpath,
                 headers=headers,
-                data=request.data,
+                data=request.data.decode(),
             )
         elif request.method == "POST":
             cellxgene_response = post(
-                full_path,
+                cellxgene_basepath + subpath,
                 headers=headers,
-                data=request.data,
+                data=request.data.decode(),
             )
         else:
             raise CellxgeneException(
@@ -187,21 +139,17 @@ class CacheEntry:
             )
         content_type = cellxgene_response.headers["content-type"]
         if "text" in content_type:
-            gateway_content = self.rewrite_text_content(
-                cellxgene_response.content.decode()
-            )
+            cellxgene_content = cellxgene_response.content.decode()
+            gateway_content = cellxgene_content.replace(
+                "http://fonts.gstatic.com", "https://fonts.gstatic.com"
+            ).replace(cellxgene_basepath, gateway_basepath)
         else:
             gateway_content = cellxgene_response.content
 
-        resp_headers = {}
-        for h in copy_headers:
-            if h in cellxgene_response.headers:
-                resp_headers[h] = cellxgene_response.headers[h]
-
         gateway_response = make_response(
             gateway_content,
             cellxgene_response.status_code,
-            resp_headers,
+            {"Content-Type": content_type},
         )
 
         return gateway_response

cellxgene_gateway/cache_key.py

@@ -1,23 +0,0 @@
-# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
-# under the Apache License, Version 2.0 (the "License"); you may not use
-# this file except in compliance with the License. You may obtain a copy
-# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
-# required by applicable law or agreed to in writing, software distributed
-# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
-# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
-# the specific language governing permissions and limitations under the License.
-
-# There are three kinds of CacheKey:
-# 1) somedir/dataset.h5ad: a dataset
-#    in this case, pathpart == dataset == 'somedir/dataset.h5ad'
-# 2) somedir/dataset_annotations/my_annotations.csv : an actual annotaitons file.
-#    in this case, pathpart == 'dataset_annotations/my_annotations.csv', dataset == 'somedir/dataset.h5ad'
-# 3) somedir/dataset_annotations: an annotation directory. The corresponding h5ad must exist, but the directory may not.
-#    in this case, pathpart == 'dataset_annotations', dataset == 'somedir/dataset.h5ad'
-
-
-class CacheKey:
-    def __init__(self, pathpart, dataset, annotation_file):
-        self.pathpart = pathpart
-        self.dataset = dataset
-        self.annotation_file = annotation_file

cellxgene_gateway/dir_util.py

@@ -52,12 +52,43 @@ def create_dir(parent_path, dir_name):
         os.mkdir(full_path)
 
 
-annotations_suffix = "_annotations"
+def recurse_dir(path):
+    if not os.path.exists(path):
+        raise CellxgeneException(
+            "The given path does not exist.", status.HTTP_400_BAD_REQUEST
+        )
+
+    def make_entry(el):
+        full_path = os.path.join(path, el)
+        if os.path.isfile(full_path):
+            return {
+                "path": full_path.replace(env.cellxgene_data, ""),
+                "name": el,
+                "type": "file",
+            }
+        elif os.path.isdir(full_path):
+            return {
+                "path": full_path,
+                "name": el,
+                "type": "directory",
+                "children": recurse_dir(full_path),
+            }
+        else:
+            raise CellxgeneException(
+                "Given path is neither file nor directory.",
+                status.HTTP_400_BAD_REQUEST,
+            )
+
+    return [make_entry(x) for x in os.listdir(path)]
 
 
-def make_h5ad(el):
-    return el[: -len(annotations_suffix)] + ".h5ad"
+def render_entries(entries):
+    return "<ul>" + "\n".join([render_entry(e) for e in entries]) + "</ul>"
 
 
-def make_annotations(el):
-    return el[:-5] + annotations_suffix
+def render_entry(entry):
+    if entry["type"] == "file":
+        url = 'view' + '/' + entry['path'].lstrip("/")
+        return f"<li> <a href='{ url}'>{entry['name']}</a></li>"
+    elif entry["type"] == "directory":
+        return f"<li>{entry['name']}{render_entries(entry['children'])}</li>"

cellxgene_gateway/env.py

@@ -7,71 +7,32 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
-import logging
 import os
-import socket
+import logging
 
 cellxgene_location = os.environ.get("CELLXGENE_LOCATION")
 cellxgene_data = os.environ.get("CELLXGENE_DATA")
-cellxgene_args = os.environ.get("CELLXGENE_ARGS", None)
-gateway_port = int(os.environ.get("GATEWAY_PORT", "5005"))
-external_host = os.environ.get(
-    "EXTERNAL_HOST",
-    os.environ.get("GATEWAY_HOST", f"localhost:{gateway_port}"),
-)
-external_protocol = os.environ.get(
-    "EXTERNAL_PROTOCOL", os.environ.get("GATEWAY_PROTOCOL", "http")
-)
-ip = os.environ.get("GATEWAY_IP", "127.0.0.1")
+gateway_host = os.environ.get("GATEWAY_HOST")
+gateway_protocol = os.environ.get("GATEWAY_PROTOCOL")
+ip = os.environ.get("GATEWAY_IP")
 extra_scripts = os.environ.get("GATEWAY_EXTRA_SCRIPTS")
 ttl = os.environ.get("GATEWAY_TTL")
-enable_upload = os.environ.get("GATEWAY_ENABLE_UPLOAD", "").lower() in [
-    "true",
-    "1",
-]
-enable_annotations = os.environ.get(
-    "GATEWAY_ENABLE_ANNOTATIONS", ""
-).lower() in [
-    "true",
-    "1",
-]
-enable_backed_mode = os.environ.get(
-    "GATEWAY_ENABLE_BACKED_MODE", ""
-).lower() in [
-    "true",
-    "1",
-]
+enable_upload = os.environ.get("GATEWAY_ENABLE_UPLOAD", "").lower() in ['true', '1']
 
 env_vars = {
     "CELLXGENE_LOCATION": cellxgene_location,
     "CELLXGENE_DATA": cellxgene_data,
+    "GATEWAY_HOST": gateway_host,
+    "GATEWAY_PROTOCOL": gateway_protocol,
+    "GATEWAY_IP": ip,
 }
 
-proxy_fix_for = int(os.environ.get("PROXY_FIX_FOR", "0"))
-proxy_fix_proto = int(os.environ.get("PROXY_FIX_PROTO", "0"))
-proxy_fix_host = int(os.environ.get("PROXY_FIX_HOST", "0"))
-proxy_fix_port = int(os.environ.get("PROXY_FIX_PORT", "0"))
-proxy_fix_prefix = int(os.environ.get("PROXY_FIX_PREFIX", "0"))
-
 optional_env_vars = {
-    "EXTERNAL_HOST": external_host,
-    "EXTERNAL_PROTOCOL": external_protocol,
-    "GATEWAY_IP": ip,
-    "GATEWAY_PORT": gateway_port,
     "GATEWAY_EXTRA_SCRIPTS": extra_scripts,
     "GATEWAY_TTL": ttl,
     "GATEWAY_ENABLE_UPLOAD": enable_upload,
-    "GATEWAY_ENABLE_ANNOTATIONS": enable_annotations,
-    "GATEWAY_ENABLE_BACKED_MODE": enable_backed_mode,
-    "CELLXGENE_ARGS": cellxgene_args,
-    "PROXY_FIX_FOR": proxy_fix_for,
-    "PROXY_FIX_PROTO": proxy_fix_proto,
-    "PROXY_FIX_HOST": proxy_fix_host,
-    "PROXY_FIX_PORT": proxy_fix_port,
-    "PROXY_FIX_PREFIX": proxy_fix_prefix,
 }
 
-
 def validate():
     if not all(env_vars.values()):
         raise ValueError(
@@ -86,12 +47,11 @@ def validate():
 
         export CELLXGENE_LOCATION=~/anaconda/envs/cellxgene-dev/bin/cellxgene
         export CELLXGENE_DATA=../cellxgene_data
+        export GATEWAY_HOST=localhost:5005
+        export GATEWAY_PROTOCOL=http
+        export GATEWAY_IP=127.0.0.1
     """
         )
     else:
-        logging.getLogger("cellxgene_gateway").info(
-            f"Got required env: {env_vars}",
-        )
-        logging.getLogger("cellxgene_gateway").info(
-            f"Got optional env: {optional_env_vars}"
-        )
+        logging.getLogger("cellxgene_gateway").info(f"Got required env: {env_vars}", )
+        logging.getLogger("cellxgene_gateway").info(f"Got optional env: {optional_env_vars}")

cellxgene_gateway/extra_scripts.py

@@ -7,14 +7,13 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
-from json import loads
-
 from cellxgene_gateway import env
+from json import loads
 
 
 def get_extra_scripts():
     # can be array of script tags to inject on every page, e.g. for google analytics could be
     # ['https://www.googletagmanager.com/gtag/js?id=UA-123456-2',
-    #  f"{env.external_protocol}://{env.external_host}/static/js/google_ua.js"]
+    #  f"{env.gateway_protocol}://{env.gateway_host}/static/js/google_ua.js"]
     # where google_ua.js is a script you add to the static/js folder prior to deployment.
     return [] if env.extra_scripts is None else loads(env.extra_scripts)

cellxgene_gateway/filecrawl.py

@@ -1,121 +0,0 @@
-# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
-# under the Apache License, Version 2.0 (the "License"); you may not use
-# this file except in compliance with the License. You may obtain a copy
-# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
-# required by applicable law or agreed to in writing, software distributed
-# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
-# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
-# the specific language governing permissions and limitations under the License.
-
-import os
-from cellxgene_gateway import env
-from cellxgene_gateway.dir_util import (
-    make_h5ad,
-    make_annotations,
-    annotations_suffix,
-)
-from flask import url_for
-
-
-def recurse_dir(path):
-    if not os.path.exists(path):
-        raise CellxgeneException(
-            "The given path does not exist.", status.HTTP_400_BAD_REQUEST
-        )
-
-    all_entries = sorted(os.listdir(path))
-
-    def is_h5ad(el):
-        return el.endswith(".h5ad") and os.path.isfile(os.path.join(path, el))
-
-    h5ad_entries = [x for x in all_entries if is_h5ad(x)]
-    annotation_dir_entries = [
-        x
-        for x in all_entries
-        if x.endswith(annotations_suffix) and make_h5ad(x) in h5ad_entries
-    ]
-
-    def list_annotations(el):
-        full_path = os.path.join(path, el)
-        if not os.path.isdir(full_path):
-            entries = []
-        else:
-            entries = [
-                {
-                    "name": x[:-13]
-                    if (len(x) > 13 and x[-13] in ["-", "_"])
-                    else (x[:-4] if x.endswith(".csv") else x),
-                    "path": os.path.join(full_path, x).replace(
-                        env.cellxgene_data, ""
-                    ),
-                }
-                for x in sorted(os.listdir(full_path))
-                if x.endswith(".csv")
-                and os.path.isfile(os.path.join(full_path, x))
-            ]
-        return [
-            {
-                "name": "new",
-                "class": "new",
-                "path": full_path.replace(env.cellxgene_data, ""),
-            }
-        ] + entries
-
-    def make_entry(el):
-        full_path = os.path.join(path, el)
-        if el in h5ad_entries:
-            return {
-                "path": full_path.replace(env.cellxgene_data, ""),
-                "name": el,
-                "type": "file",
-                "annotations": list_annotations(make_annotations(el)),
-            }
-        elif os.path.isdir(full_path) and el not in annotation_dir_entries:
-            return {
-                "path": full_path.replace(env.cellxgene_data, ""),
-                "name": el,
-                "type": "directory",
-                "children": recurse_dir(full_path),
-            }
-        else:
-            return {
-                "path": full_path,
-                "name": el,
-                "type": "neither",
-            }
-
-    return [make_entry(x) for x in all_entries]
-
-
-def render_entries(entries):
-    return "<ul>" + "\n".join([render_entry(e) for e in entries]) + "</ul>"
-
-
-def get_url(entry):
-    return url_for("do_view", path=entry["path"].lstrip("/") + "/")
-
-
-def get_class(entry):
-    return f" class='{entry['class']}'" if "class" in entry else ""
-
-
-def render_annotations(entry):
-    if len(entry["annotations"]) > 0:
-        return " | annotations: " + ", ".join(
-            [
-                f"<a href='{get_url(a)}'{get_class(a)}>{a['name']}</a>"
-                for a in entry["annotations"]
-            ]
-        )
-    else:
-        return ""
-
-
-def render_entry(entry):
-    if entry["type"] == "file":
-        return f"<li> <a href='{ get_url(entry) }'>{entry['name']}</a> {render_annotations(entry)}</li>"
-    elif entry["type"] == "directory":
-        url = f"/filecrawl/{entry['path'].lstrip('/')}"
-        return f"<li><a href='{url}'>{entry['name']}</a>{render_entries(entry['children'])}</li>"
-    else:
-        return ""

cellxgene_gateway/flask_util.py

@@ -1,15 +0,0 @@
-# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
-# under the Apache License, Version 2.0 (the "License"); you may not use
-# this file except in compliance with the License. You may obtain a copy
-# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
-# required by applicable law or agreed to in writing, software distributed
-# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
-# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
-# the specific language governing permissions and limitations under the License.
-
-from flask import request
-
-
-def querystring():
-    qs = request.query_string.decode()
-    return f"?{qs}" if len(qs) > 0 else ""

cellxgene_gateway/gateway.py

@@ -7,16 +7,15 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
-import json
-import logging
-
 # import BaseHTTPServer
+import datetime
 import os
-from threading import Lock, Thread
+import logging
+from threading import Thread, Lock
+import json
 
 from flask import (
     Flask,
-    make_response,
     redirect,
     render_template,
     request,
@@ -24,49 +23,21 @@ from flask import (
     url_for,
 )
 from flask_api import status
-from werkzeug.utils import secure_filename
-from werkzeug.middleware.proxy_fix import ProxyFix
+from werkzeug import secure_filename
+
 from cellxgene_gateway import env
 from cellxgene_gateway.backend_cache import BackendCache
-from cellxgene_gateway.cache_entry import CacheEntryStatus
 from cellxgene_gateway.cellxgene_exception import CellxgeneException
-from cellxgene_gateway.dir_util import create_dir, is_subdir
+from cellxgene_gateway.dir_util import create_dir, recurse_dir, render_entries, is_subdir
 from cellxgene_gateway.extra_scripts import get_extra_scripts
-from cellxgene_gateway.filecrawl import recurse_dir, render_entries
-from cellxgene_gateway.path_util import get_key
+from cellxgene_gateway.path_util import get_dataset, get_file_path
 from cellxgene_gateway.process_exception import ProcessException
 from cellxgene_gateway.prune_process_cache import PruneProcessCache
 from cellxgene_gateway.util import current_time_stamp
 
 app = Flask(__name__)
-
-
-def _force_https(app):
-    def wrapper(environ, start_response):
-        environ["wsgi.url_scheme"] = env.external_protocol
-        return app(environ, start_response)
-
-    return wrapper
-
-
-app.wsgi_app = _force_https(app.wsgi_app)
-if (
-    env.proxy_fix_for > 0
-    or env.proxy_fix_proto > 0
-    or env.proxy_fix_host > 0
-    or env.proxy_fix_port > 0
-    or env.proxy_fix_prefix > 0
-):
-    app.wsgi_app = ProxyFix(
-        app.wsgi_app,
-        x_for=env.proxy_fix_for,
-        x_proto=env.proxy_fix_proto,
-        x_host=env.proxy_fix_host,
-        x_port=env.proxy_fix_port,
-        x_prefix=env.proxy_fix_prefix,
-    )
-
 cache = BackendCache()
+location = f"{env.gateway_protocol}://{env.gateway_host}"
 
 
 @app.errorhandler(CellxgeneException)
@@ -102,8 +73,7 @@ def handle_invalid_process(error):
             http_status=error.http_status,
             stdout=error.stdout,
             stderr=error.stderr,
-            dataset=error.key.dataset,
-            annotation_file=error.key.annotation_file,
+            dataset=error.dataset,
         ),
         error.http_status,
     )
@@ -134,13 +104,12 @@ def index():
         enable_upload=env.enable_upload,
     )
 
-
 def make_user():
     dir_name = request.form["directory"]
 
     create_dir(env.cellxgene_data, dir_name)
 
-    return redirect(url_for("index"), code=302)
+    return redirect(location, code=302)
 
 
 def make_subdir():
@@ -149,26 +118,22 @@ def make_subdir():
 
     create_dir(parent_path, dir_name)
 
-    return redirect(url_for("index"), code=302)
+    return redirect(location, code=302)
 
 
 def upload_file():
     upload_dir = request.form["path"]
 
     full_upload_path = os.path.join(env.cellxgene_data, upload_dir)
-    if is_subdir(full_upload_path, env.cellxgene_data) and os.path.isdir(
-        full_upload_path
-    ):
+    if is_subdir(full_upload_path, env.cellxgene_data) and os.path.isdir(full_upload_path):
         if request.method == "POST":
             if "file" in request.files:
                 f = request.files["file"]
                 if f and f.filename.endswith(".h5ad"):
                     f.save(
-                        os.path.join(
-                            full_upload_path, secure_filename(f.filename)
-                        )
+                        os.path.join(full_upload_path, secure_filename(f.filename))
                     )
-                    return redirect(url_for("filecrawl"), code=302)
+                    return redirect("/filecrawl.html", code=302)
                 else:
                     raise CellxgeneException(
                         "Uploaded file must be in anndata (.h5ad) format.",
@@ -184,82 +149,46 @@ def upload_file():
             "Invalid directory.", status.HTTP_400_BAD_REQUEST
         )
 
-    return redirect(url_for("index"), code=302)
+    return redirect(env.location, code=302)
 
 
 if env.enable_upload:
-    app.add_url_rule("/make_user", "make_user", make_user, methods=["POST"])
-    app.add_url_rule(
-        "/make_subdir", "make_subdir", make_subdir, methods=["POST"]
-    )
-    app.add_url_rule(
-        "/upload_file", "upload_file", upload_file, methods=["POST"]
-    )
-
-
-def set_no_cache(resp):
-    resp.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
-    resp.headers["Pragma"] = "no-cache"
-    resp.headers["Expires"] = "0"
-    resp.headers["Cache-Control"] = "public, max-age=0"
-    return resp
-
+    app.add_url_rule('/make_user', 'make_user', make_user, methods=["POST"])
+    app.add_url_rule('/make_subdir', 'make_subdir', make_subdir, methods=["POST"])
+    app.add_url_rule('/upload_file', 'upload_file', upload_file, methods=["POST"])
 
 @app.route("/filecrawl.html")
 def filecrawl():
+
     entries = recurse_dir(env.cellxgene_data)
     rendered_html = render_entries(entries)
-    resp = make_response(
-        render_template(
-            "filecrawl.html",
-            extra_scripts=get_extra_scripts(),
-            rendered_html=rendered_html,
-        )
-    )
-    return set_no_cache(resp)
-
-
-@app.route("/filecrawl/<path:path>")
-def do_filecrawl(path):
-    filecrawl_path = os.path.join(env.cellxgene_data, path)
-    if not os.path.isdir(filecrawl_path):
-        raise CellxgeneException(
-            "Path is not directory: " + filecrawl_path,
-            status.HTTP_400_BAD_REQUEST,
-        )
-    entries = recurse_dir(filecrawl_path)
-    rendered_html = render_entries(entries)
     return render_template(
         "filecrawl.html",
         extra_scripts=get_extra_scripts(),
         rendered_html=rendered_html,
-        path=path,
     )
 
-
 entry_lock = Lock()
-
-
 @app.route("/view/<path:path>", methods=["GET", "PUT", "POST"])
 def do_view(path):
-    key = get_key(path)
-    print(
-        f"view path={path}, dataset={key.dataset}, annotation_file= {key.annotation_file}, key={key.pathpart}"
-    )
+    dataset = get_dataset(path)
+    file_path = get_file_path(dataset)
     with entry_lock:
-        match = cache.check_entry(key)
+        match = cache.check_entry(dataset)
         if match is None:
             uascripts = get_extra_scripts()
-            match = cache.create_entry(key, uascripts)
+            match = cache.create_entry(dataset, file_path, uascripts)
 
     match.timestamp = current_time_stamp()
 
-    if (
-        match.status == CacheEntryStatus.loaded
-        or match.status == CacheEntryStatus.loading
-    ):
+    if match.status == "loaded":
         return match.serve_content(path)
-    elif match.status == CacheEntryStatus.error:
+    elif match.status == "loading":
+        launch_time = datetime.datetime.fromtimestamp(match.launchtime)
+        return render_template(
+            "loading.html", launchtime=launch_time, all_output=match.all_output
+        )
+    elif match.status == "error":
         raise ProcessException.from_cache_entry(match)
 
 
@@ -267,59 +196,35 @@ def do_view(path):
 def do_GET_status():
     return render_template("cache_status.html", entry_list=cache.entry_list)
 
-
 @app.route("/cache_status.json", methods=["GET"])
 def do_GET_status_json():
-    return json.dumps(
-        {
-            "launchtime": app.launchtime,
-            "entry_list": [
-                {
-                    "dataset": entry.key.dataset,
-                    "annotation_file": entry.key.annotation_file,
-                    "launchtime": entry.launchtime,
-                    "last_access": entry.timestamp,
-                    "status": entry.status,
-                }
-                for entry in cache.entry_list
-            ],
-        }
-    )
-
+    return json.dumps({'launchtime':app.launchtime,
+    'entry_list':[{
+        'dataset': entry.dataset,
+        'launchtime': entry.launchtime,
+        'last_access': entry.timestamp,
+        'status': entry.status
+    } for entry in cache.entry_list]})
 
 @app.route("/relaunch/<path:path>", methods=["GET"])
 def do_relaunch(path):
-    key = get_key(path)
-    match = cache.check_entry(key)
+    dataset = get_dataset(path)
+    match = cache.check_entry(dataset)
     if not match is None:
         match.terminate()
-    qs = request.query_string.decode()
-    return redirect(
-        url_for("do_view", path=path) + (f"?{qs}" if len(qs) > 0 else ""),
-        code=302,
-    )
-
+    return redirect(url_for("do_view", path=path), code=302)
 
 @app.route("/terminate/<path:path>", methods=["GET"])
 def do_terminate(path):
-    key = get_key(path)
-    match = cache.check_entry(key)
+    dataset = get_dataset(path)
+    match = cache.check_entry(dataset)
     if not match is None:
         match.terminate()
     return redirect(url_for("do_GET_status"), code=302)
 
 
-@app.route("/metadata/ip_address", methods=["GET"])
-def ip_address():
-    resp = make_response(env.ip)
-    return set_no_cache(resp)
-
-
 def main():
-    logging.basicConfig(
-        level=logging.INFO,
-        format="%(asctime)s:%(name)s:%(levelname)s:%(message)s",
-    )
+    logging.basicConfig(level=logging.INFO, format='%(asctime)s:%(name)s:%(levelname)s:%(message)s')
     env.validate()
     pruner = PruneProcessCache(cache)
 
@@ -327,7 +232,7 @@ def main():
     background_thread.start()
 
     app.launchtime = current_time_stamp()
-    app.run(host="0.0.0.0", port=env.gateway_port, debug=False)
+    app.run(host="0.0.0.0", port=5005, debug=False)
 
 
 if __name__ == "__main__":

cellxgene_gateway/path_util.py

@@ -12,51 +12,30 @@ import os
 from flask_api import status
 
 from cellxgene_gateway import env
-from cellxgene_gateway.cache_key import CacheKey
 from cellxgene_gateway.cellxgene_exception import CellxgeneException
-from cellxgene_gateway.dir_util import make_h5ad
 
 
-def get_key(path):
+def get_dataset(path):
     if path == "/" or path == "":
         raise CellxgeneException(
             "No matching dataset found.", status.HTTP_404_NOT_FOUND
         )
 
     trimmed = path[:-1] if path[-1] == "/" else path
+
     try:
-        # valid paths come in three forms:
-        if trimmed.endswith(".h5ad") and data_file_exists(trimmed):
-            # 1) somedir/dataset.h5ad: a dataset
-            return CacheKey(trimmed, trimmed, None)
-        elif trimmed.endswith(".csv"):
-
-            # 2) somedir/dataset_annotations/my_annotations.csv : an actual annotations file.
-            annotations_dir = os.path.split(trimmed)[0]
-            dataset = make_h5ad(annotations_dir)
-            if data_file_exists(dataset):
-                data_dir_ensure(annotations_dir)
-                return CacheKey(trimmed, dataset, trimmed)
-        elif trimmed.endswith("_annotations") and data_dir_exists(trimmed):
-            # 3) somedir/dataset_annotations: an annotation directory. The corresponding h5ad must exist, but the directory may not.
-            dataset = make_h5ad(trimmed)
-            if data_file_exists(dataset):
-                return CacheKey(trimmed, dataset, "")
+        get_file_path(trimmed)
+        return trimmed
     except CellxgeneException:
-        pass
-    split = os.path.split(trimmed)
-    return get_key(split[0])
+        split = os.path.split(trimmed)
+        return get_dataset(split[0])
 
 
-def validate_exists(file_path):
+def validate_path(file_path):
     if not os.path.exists(file_path):
         raise CellxgeneException(
             "File does not exist: " + file_path, status.HTTP_400_BAD_REQUEST
         )
-
-
-def validate_is_file(file_path):
-    validate_exists(file_path)
     if not os.path.isfile(file_path):
         raise CellxgeneException(
             "Path is not file: " + file_path, status.HTTP_400_BAD_REQUEST
@@ -64,44 +43,7 @@ def validate_is_file(file_path):
     return
 
 
-def validate_is_dir(file_path):
-    validate_exists(file_path)
-    if not os.path.isdir(file_path):
-        raise CellxgeneException(
-            "Path is not dir: " + file_path, status.HTTP_400_BAD_REQUEST
-        )
-    return
-
-
-def data_file_exists(dataset):
+def get_file_path(dataset):
     file_path = os.path.join(env.cellxgene_data, dataset)
-    validate_is_file(file_path)
-    return True
-
-
-def data_dir_exists(dataset):
-    file_path = os.path.join(env.cellxgene_data, dataset)
-    validate_is_dir(file_path)
-    return True
-
-
-def data_dir_ensure(dataset):
-    file_path = os.path.join(env.cellxgene_data, dataset)
-    if not os.path.exists(file_path):
-        os.makedirs(file_path)
-
-
-def get_file_path(key):
-    dataset = key.dataset
-    file_path = os.path.join(env.cellxgene_data, dataset)
-    validate_is_file(file_path)
-    return file_path
-
-
-def get_annotation_file_path(key):
-    if key.annotation_file is None:
-        return None
-    if key.annotation_file == "":
-        return ""
-    file_path = os.path.join(env.cellxgene_data, key.annotation_file)
+    validate_path(file_path)
     return file_path

cellxgene_gateway/process_exception.py

@@ -9,13 +9,13 @@
 
 
 class ProcessException(Exception):
-    def __init__(self, message, stdout, stderr, http_status, key):
+    def __init__(self, message, stdout, stderr, http_status, dataset):
         Exception.__init__(self)
         self.message = message
         self.stdout = stdout
         self.stderr = stderr
         self.http_status = http_status
-        self.key = key
+        self.dataset = dataset
 
     @classmethod
     def from_cache_entry(cls, cache_entry):
@@ -24,5 +24,5 @@ class ProcessException(Exception):
             cache_entry.all_output,
             cache_entry.stderr,
             cache_entry.http_status,
-            cache_entry.key,
+            cache_entry.dataset,
         )

cellxgene_gateway/prune_process_cache.py

@@ -7,19 +7,16 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
-import logging
 import time
+import logging
 
-from cellxgene_gateway import env
-from cellxgene_gateway import util
-
-logger = logging.getLogger(__name__)
-
+from cellxgene_gateway.util import current_time_stamp
+from cellxgene_gateway.env import ttl
 
 class PruneProcessCache:
     def __init__(self, cache):
         self.cache = cache
-        self.expire_seconds = 3600 if env.ttl is None else int(env.ttl)
+        self.expire_seconds = (3600 if ttl is None else int(ttl))
 
     def __call__(self):
         while True:
@@ -27,26 +24,18 @@ class PruneProcessCache:
             self.prune()
 
     def prune(self):
-        timestamp = util.current_time_stamp()
+        timestamp = current_time_stamp()
         cutoff = timestamp - self.expire_seconds
-        processes_to_delete = [
-            p for p in self.cache.entry_list if p.timestamp < cutoff
-        ]
-        processes_to_keep = [
-            p for p in self.cache.entry_list if not p.timestamp < cutoff
-        ]
-
-        logger.debug(
-            f"Cutoff {cutoff} = timestamp {timestamp} - expire seconds {self.expire_seconds} , keeping {processes_to_keep}, pruning {processes_to_delete}"
-        )
-
+        def prunable(p):
+            return p.timestamp < cutoff and p.pid != None
+        processes_to_delete = [p for p in self.cache.entry_list if  prunable(p)]
+        processes_to_keep = [p for p in self.cache.entry_list if not prunable(p)]
+        logger = logging.getLogger("cellxgene_gateway")
+        logger.debug(f"Cutoff {cutoff} = timestamp {timestamp} - expire seconds {self.expire_seconds} , keeping {processes_to_keep}")
+        
         for process in processes_to_delete:
             try:
-                logger.info(
-                    f"pruning process {process.pid} ({process.key.dataset})"
-                )
+                logger.info(f"pruning process {process.pid} ({process.dataset})")
                 self.cache.prune(process)
             except Exception:
-                logger.exception(
-                    "failed to prune process {process.pid} ({process.dataset})"
-                )
+                logger.exception("failed to prune process {process.pid} ({process.dataset})")

cellxgene_gateway/static/js/annotation.js

@@ -1,19 +0,0 @@
-// neandertal javascript
-const new_annotation_callback = (() =>{
-    const suffix = `.csv`;
-    return (e) => {
-        e.preventDefault();
-        const el = $(e.target);
-        const href = el.attr('href');
-        const base = prompt(`Name your annotations collection\nnote: the suffix "${suffix}" will be appended`);
-        if (base !== null && base.length > 0) {
-            if (/^[0-9a-zA-Z_]+$/.test(base)) {
-                window.location = `${href}/${base}${suffix}`;
-            } else {
-                alert("Error: name must match ^[0-9a-zA-Z_]+$\nthat is, only numbers, letters and underscore are allowed")
-            }
-        }
-        return false;
-    }
-})()
-

cellxgene_gateway/subprocess_backend.py

@@ -11,14 +11,7 @@ import logging
 import subprocess
 
 from flask_api import status
-from cellxgene_gateway.cache_entry import CacheEntryStatus
-from cellxgene_gateway.dir_util import make_annotations
-from cellxgene_gateway.path_util import get_annotation_file_path, get_file_path
-from cellxgene_gateway.env import (
-    enable_annotations,
-    enable_backed_mode,
-    cellxgene_args,
-)
+
 from cellxgene_gateway.process_exception import ProcessException
 
 
@@ -26,29 +19,13 @@ class SubprocessBackend:
     def __init__(self):
         pass
 
-    def create_cmd(
-        self, cellxgene_loc, file_path, port, scripts, annotation_file_path
-    ):
-        if enable_annotations and not annotation_file_path is None:
-            if annotation_file_path == "":
-                extra_args = (
-                    f" --annotations-dir {make_annotations(file_path)}"
-                )
-            else:
-                extra_args = f" --annotations-file {annotation_file_path}"
-        else:
-            extra_args = " --disable-annotations"
-        if enable_backed_mode:
-            extra_args += " --backed"
-        if not cellxgene_args is None:
-            extra_args += f" {cellxgene_args}"
+    def create_cmd(self, cellxgene_loc, file_path, port, scripts):
 
         cmd = (
             f"yes | {cellxgene_loc} launch {file_path}"
             + " --port "
             + str(port)
             + " --host 127.0.0.1"
-            + extra_args
         )
 
         for s in scripts:
@@ -59,11 +36,7 @@ class SubprocessBackend:
     def launch(self, cellxgene_loc, scripts, cache_entry):
 
         cmd = self.create_cmd(
-            cellxgene_loc,
-            get_file_path(cache_entry.key),
-            cache_entry.port,
-            scripts,
-            get_annotation_file_path(cache_entry.key),
+            cellxgene_loc, cache_entry.file_path, cache_entry.port, scripts
         )
         logging.getLogger("cellxgene_gateway").info(f"launching {cmd}")
         process = subprocess.Popen(
@@ -86,7 +59,7 @@ class SubprocessBackend:
                     message = "Cellxgene failed to launch dataset."
                     http_status = status.HTTP_500_INTERNAL_SERVER_ERROR
 
-                cache_entry.status = CacheEntryStatus.error
+                cache_entry.status = "error"
                 cache_entry.set_error(message, stderr, http_status)
 
                 raise ProcessException.from_cache_entry(cache_entry)

cellxgene_gateway/templates/cache_status.html

@@ -29,7 +29,6 @@
           <tr>
                <th>PID</th>
                <th>dataset</th>
-               <th>annotation_file</th>
                <th>port</th>
                <th>launchtime</th>
                <th>last access</th>
@@ -43,17 +42,16 @@
 	{% for entry in entry_list %}
           <tr>
                <td>{{ entry.pid }}</td>
-               <td><a href="{{ url_for('do_view', path=entry.key.pathpart) }}">{{ entry.key.dataset }}</a></td>
-               <td>{{ entry.key.annotation_file }}</td>
+               <td><a href="{{ url_for('do_view', path=entry.dataset) }}">{{ entry.dataset }}</a></td>
                <td>{{ entry.port }}</td>
                <td class="timestamp">{{ entry.launchtime }}</td>
                <td class="timestamp">{{ entry.timestamp }}</td>
-               <td>{{ entry.status.name }}</td>
+               <td>{{ entry.status }}</td>
                <td>{{ entry.message }}</td>
                <td>{{ entry.http_status }}</td>
                <td>
-                 {% if entry.status.name == 'loaded' %}
-                   <a href="{{ url_for('do_terminate', path=entry.key.pathpart) }}"> terminate </a>
+                 {% if entry.status == 'loaded' %}
+                   <a href="{{ url_for('do_terminate', path=entry.dataset) }}"> terminate </a>
                  {% endif %}
                </td>
           </tr>

cellxgene_gateway/templates/cellxgene_error.html

@@ -28,11 +28,11 @@
           
      <h4>{{ message }}</h4>
 
-     <a href="{{ url_for('filecrawl') }}">
+     <a href="/filecrawl.html">
           Please click here to be redirected to the file directory.
      </a>
      <br>
-     <a href="{{ url_for('index') }}">
+     <a href="/">
           Please click here to return to the homepage.
      </a>
      </div>

cellxgene_gateway/templates/filecrawl.html

@@ -16,36 +16,19 @@
      <link rel="icon" type="image/png" href="{{ url_for('static', filename='nibr.ico') }}">
 	{% for script in extra_scripts %}
 	<script src="{{ script }}"></script>
-     {% endfor %}
-     <script src="{{ url_for('static', filename='js/annotation.js') }}"></script>
+	{% endfor %}
      <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css" integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO" crossorigin="anonymous">
 </head>
 <body>
      <header class="navbar navbar-expand navbar-dark flex-column flex-md-row bd-navbar">
-          {% if path %}
-          <h3>Cellxgene Gateway - {{ path }}</h3>
-          {% else %}
-          <h3>Cellxgene Gateway - FILE CRAWLER</h3>
-          {% endif %}
+       <h3>Cellxgene Gateway - FILE CRAWLER</h3>
      </header>
+     <br>
 
      <h4>Please click on a dataset to view it in Cellxgene Server.</h4>
 
+     <br>
      {{ rendered_html|safe }}
-     <p>
-          Navigation:
-     <ul>
-     {% if path %}
-     <li><a href="{{ url_for('filecrawl') }}">top level</a></li>
-     {% else %}
-     {% endif %}
-     <li><a href="{{ url_for('index') }}">homepage</a></li>
-     </ul>
-     </p>
-     <script>
-          $(() => {
-               $("a.new").click(new_annotation_callback);
-          })
-     </script>
+
 </body>
 </html>

cellxgene_gateway/templates/index.html

@@ -35,12 +35,12 @@
 		Links:	
 	</h1>
 	<div class="list-group" style="width:50%;padding-left:65px">
-	  <a href="{{ url_for('filecrawl') }}" class="list-group-item list-group-item-action">
+	  <a href="/filecrawl.html" class="list-group-item list-group-item-action">
 		  <u>File Crawler: Allows you to view all uploaded data.</u></a>
 		  
 	</div>
 	<div class="list-group" style="width:50%;padding-left:65px">
-		<a href="{{ url_for('do_GET_status') }}" class="list-group-item list-group-item-action">
+		<a href="/cache_status" class="list-group-item list-group-item-action">
 			<u>Cache Status: view status of launched cellxgene servers.</u></a>
 	</div>
 	

cellxgene_gateway/templates/loading.html

@@ -35,22 +35,18 @@
           The page will refresh shortly.
      </p>
 
-     <a href="{{ url_for('filecrawl') }}">
+     <a href="/filecrawl.html">
           Please click here to be redirected to the file directory.
      </a>
      <br>
-     <a href="{{ url_for('index') }}">
+     <a href="/">
           Please click here to return to the homepage.
      </a>
      </div>
      <script>
-          var count = 0;
           window.setInterval(function(){
                var dots = document.getElementById('dots');
                dots.textContent = dots.textContent + '.';
-               if (count++ > 5) {
-                    window.location.reload();
-               }
           }, 1000);
      </script>
 </body>

cellxgene_gateway/templates/process_error.html

@@ -39,10 +39,10 @@
           <li><a href="{{url_for('do_relaunch', path=dataset)}}">
                     Attempt to relaunch the cellxgene server.
                </a></li>
-          <li><a href="{{ url_for('filecrawl') }}">
+          <li><a href="/filecrawl.html">
                Return to the file directory.
           </a></li>
-          <li><a href="{{ url_for('index') }}">
+          <li><a href="/">
                Return to the homepage.
           </a></li>
      </ul>

environment-dev.yml

@@ -1,4 +1,4 @@
-name: cellxgene-gateway
+name: cellxgene-dev
 channels:
 - conda-forge
 dependencies:
@@ -6,8 +6,6 @@ dependencies:
 - requests
 - flask
 - psutil
-- black
-- pip
 - pip:
   - flask-api
-  - cellxgene>=0.15
+  - cellxgene

requirements.txt

@@ -1,4 +1,4 @@
-cellxgene>=0.15
+cellxgene
 flask
 flask_api
 psutil

run.sh.example

@@ -1,5 +1,8 @@
 export CELLXGENE_LOCATION=$(pwd)/.cellxgene-gateway/bin/cellxgene
 export CELLXGENE_DATA=../cellxgene_data
+export DEPLOYMENT_ENV=dev
+export GATEWAY_HOST=localhost:5005
+export GATEWAY_PROTOCOL=http
 export GATEWAY_IP=127.0.0.1
 
 #Once these are set, you run like a normal Flask app

setup.cfg

@@ -1,2 +0,0 @@
-[metadata]
-description-file = README.md

setup.py

@@ -1,25 +1,5 @@
 import os
-import codecs
-from setuptools import find_packages, setup
-import sys
-
-if sys.version_info < (3, 6):
-    sys.exit("Sorry, Python < 3.6 is not supported")
-
-
-def read(rel_path):
-    here = os.path.abspath(os.path.dirname(__file__))
-    with codecs.open(os.path.join(here, rel_path), "r") as fp:
-        return fp.read()
-
-
-def get_version(rel_path):
-    for line in read(rel_path).splitlines():
-        if line.startswith("__version__"):
-            delim = '"' if '"' in line else "'"
-            return line.split(delim)[1]
-    else:
-        raise RuntimeError("Unable to find version string.")
+from setuptools import setup
 
 
 def parse_requirements():
@@ -30,22 +10,17 @@ def parse_requirements():
     return reqs
 
 
-with open("README.md", "r") as fh:
-    long_description = fh.read()
-
 install_reqs = parse_requirements()
 
 setup(
     # mandatory
     name="cellxgene-gateway",
     # mandatory
-    version=get_version("cellxgene_gateway/__init__.py"),
+    version="0.1",
     # mandatory
     author="Niket Patel, Yohann Potier, Alok Saldanha",
     author_email="alok.saldanha@novartis.com",
     description=("Cellxgene Gateway"),
-    long_description=long_description,
-    long_description_content_type="text/markdown",
     license="MIT",
     keywords="visualization, genomics",
     url="http://github.com/Novartis/cellxgene-gateway",
@@ -53,16 +28,13 @@ setup(
     package_data={
         "cellxgene_gateway": [
             "static/css/homepagestyle.css",
-            "static/js/annotation.js",
             "static/nibr.ico",
-            "templates/*.html",
-        ]
-    },
-    data_files=[("", ["README.md", "LICENSE"])],
+            "templates/*.html"
+    ]},
+    data_files=[('', ['Readme.md', 'LICENSE.txt'])],
     install_requires=install_reqs,
     entry_points={
         "console_scripts": ["cellxgene-gateway=cellxgene_gateway.gateway:main"]
     },
     classifiers=["Topic :: Scientific/Engineering :: Visualization"],
-    python_requires=">=3.6",
 )

tests/test_cache_entry.py

@@ -1,37 +0,0 @@
-import unittest
-from flask import Flask
-from cellxgene_gateway.cache_entry import CacheEntry, CacheEntryStatus
-from cellxgene_gateway.cache_key import CacheKey
-from cellxgene_gateway.gateway import app
-
-key = CacheKey("czi/pbmc3k.h5ad", "pbmc3k.h5ad", "tmp.csv")
-
-
-class TestRenderEntry(unittest.TestCase):
-    def setUp(self):
-        self.app = app
-        self.app_context = self.app.test_request_context()
-        self.app_context.push()
-        self.client = self.app.test_client()
-
-    def test_GIVEN_key_and_port_THEN_returns_loading_CacheEntry(self):
-        entry = CacheEntry.for_key("some-key", 1)
-        self.assertEqual(entry.status, CacheEntryStatus.loading)
-
-    def test_GIVEN_absolute_static_url_THEN_include_path(self):
-        actual = CacheEntry.for_key(key, 8000).rewrite_text_content(
-            "src:url(/static/assets/"
-        )
-        expected = "src:url(/view/czi/pbmc3k.h5ad/static/assets/"
-        self.assertEqual(actual, expected)
-
-    def test_GIVEN_absolute_src_THEN_include_path(self):
-        actual = CacheEntry.for_key(key, 8000).rewrite_text_content(
-            '<link rel="shortcut icon" href="/static/assets/favicon.ico">'
-        )
-        expected = '<link rel="shortcut icon" href="/view/czi/pbmc3k.h5ad/static/assets/favicon.ico">'
-        self.assertEqual(actual, expected)
-
-
-if __name__ == "__main__":
-    unittest.main()

tests/test_dir_util.py

@@ -1,49 +1,38 @@
 import unittest
 from unittest.mock import MagicMock, patch
-from cellxgene_gateway.filecrawl import render_entry
-
+from cellxgene_gateway.dir_util import render_entry
 
 class TestRenderEntry(unittest.TestCase):
     def test_GIVEN_path_both_slash_THEN_view_has_single_slash(self):
         entry = {
-            "path": "/somepath/",
-            "name": "entry",
-            "type": "file",
-            "annotations": [],
-            "children": [],
-        }
+                "path": "/somepath/",
+                "name": "entry",
+                "type": "file",
+            }
         rendered = render_entry(entry)
-        self.assertIn("view/somepath", rendered)
-
+        self.assertIn('view/somepath', rendered)
     def test_GIVEN_path_starts_slash_THEN_view_has_single_slash(self):
         entry = {
-            "path": "/somepath",
-            "name": "entry",
-            "type": "file",
-            "annotations": [],
-            "children": [],
-        }
+                "path": "/somepath",
+                "name": "entry",
+                "type": "file",
+            }
         rendered = render_entry(entry)
-        self.assertIn("view/somepath", rendered)
-
+        self.assertIn('view/somepath', rendered)
     def test_GIVEN_path_ends_slash_THEN_view_has_single_slash(self):
         entry = {
-            "path": "somepath/",
-            "name": "entry",
-            "type": "file",
-            "annotations": [],
-            "children": [],
-        }
+                "path": "somepath/",
+                "name": "entry",
+                "type": "file",
+            }
         rendered = render_entry(entry)
-        self.assertIn("view/somepath", rendered)
-
+        self.assertIn('view/somepath', rendered)
     def test_GIVEN_path_no_slash_THEN_view_has_single_slash(self):
         entry = {
-            "path": "somepath",
-            "name": "entry",
-            "type": "file",
-            "annotations": [],
-            "children": [],
-        }
+                "path": "somepath",
+                "name": "entry",
+                "type": "file",
+            }
         rendered = render_entry(entry)
-        self.assertIn("view/somepath", rendered)
+        self.assertIn('view/somepath', rendered)
+        

tests/test_extra_scripts.py

@@ -1,26 +1,23 @@
 import unittest
 from unittest.mock import MagicMock, patch
-
 from cellxgene_gateway.extra_scripts import get_extra_scripts
 
-
 class TestExtraScripts(unittest.TestCase):
-    @patch("cellxgene_gateway.env.extra_scripts", new='["abc","def"]')
+    @patch('cellxgene_gateway.env.extra_scripts', new='["abc","def"]')
     def test_GIVEN_two_scripts_THEN_returns_two_strings(self):
-        self.assertEqual(get_extra_scripts(), ["abc", "def"])
+        self.assertEqual(get_extra_scripts(), ['abc', 'def'])
 
-    @patch("cellxgene_gateway.env.extra_scripts", new='["abc", "def"]')
+    @patch('cellxgene_gateway.env.extra_scripts', new='["abc", "def"]')
     def test_GIVEN_two_scripts_space_THEN_returns_two_strings(self):
-        self.assertEqual(get_extra_scripts(), ["abc", "def"])
+        self.assertEqual(get_extra_scripts(), ['abc', 'def'])
 
-    @patch("cellxgene_gateway.env.extra_scripts", new=None)
+    @patch('cellxgene_gateway.env.extra_scripts', new=None)
     def test_GIVEN_none_THEN_returns_empty_array(self):
         self.assertEqual(get_extra_scripts(), [])
 
-    @patch("cellxgene_gateway.env.extra_scripts", new="[]")
+    @patch('cellxgene_gateway.env.extra_scripts', new='[]')
     def test_GIVEN_empty_string_THEN_returns_empty_array(self):
         self.assertEqual(get_extra_scripts(), [])
 
-
-if __name__ == "__main__":
-    unittest.main()
+if __name__ == '__main__':
+    unittest.main()

tests/test_filecrawl.py

@@ -1,46 +0,0 @@
-import unittest
-from unittest.mock import MagicMock, patch
-
-from cellxgene_gateway.filecrawl import render_entry
-
-
-class TestRenderEntry(unittest.TestCase):
-    def test_GIVEN_path_both_slash_THEN_view_has_single_slash(self):
-        entry = {
-            "path": "/somepath/",
-            "name": "entry",
-            "type": "file",
-            "annotations": [],
-        }
-        rendered = render_entry(entry)
-        self.assertIn("view/somepath", rendered)
-
-    def test_GIVEN_path_starts_slash_THEN_view_has_single_slash(self):
-        entry = {
-            "path": "/somepath",
-            "name": "entry",
-            "type": "file",
-            "annotations": [],
-        }
-        rendered = render_entry(entry)
-        self.assertIn("view/somepath", rendered)
-
-    def test_GIVEN_path_ends_slash_THEN_view_has_single_slash(self):
-        entry = {
-            "path": "somepath/",
-            "name": "entry",
-            "type": "file",
-            "annotations": [],
-        }
-        rendered = render_entry(entry)
-        self.assertIn("view/somepath", rendered)
-
-    def test_GIVEN_path_no_slash_THEN_view_has_single_slash(self):
-        entry = {
-            "path": "somepath",
-            "name": "entry",
-            "type": "file",
-            "annotations": [],
-        }
-        rendered = render_entry(entry)
-        self.assertIn("view/somepath", rendered)

tests/test_prune_process_cache.py

@@ -1,15 +1,13 @@
 import unittest
 from unittest.mock import MagicMock, patch
-
-from cellxgene_gateway.backend_cache import BackendCache
 from cellxgene_gateway.cache_entry import CacheEntry
-
+from cellxgene_gateway.backend_cache import BackendCache
 
 class TestPruneProcessCache(unittest.TestCase):
-    @patch("cellxgene_gateway.util.current_time_stamp", new=lambda: 0)
-    @patch("cellxgene_gateway.env.ttl", new="10")
-    @patch("cellxgene_gateway.cache_entry.CacheEntry")
-    @patch("cellxgene_gateway.cache_entry.CacheEntry")
+    @patch('cellxgene_gateway.util.current_time_stamp', new=lambda:0)
+    @patch('cellxgene_gateway.env.ttl', new='10')
+    @patch('cellxgene_gateway.cache_entry.CacheEntry')
+    @patch('cellxgene_gateway.cache_entry.CacheEntry')
     def test_GIVEN_one_old_one_new_THEN_prune_old(self, old, new):
         from cellxgene_gateway.prune_process_cache import PruneProcessCache
 
@@ -24,6 +22,5 @@ class TestPruneProcessCache(unittest.TestCase):
         self.assertEqual(len(cache.entry_list), 1)
         self.assertEqual(cache.entry_list[0], new)
 
-
-if __name__ == "__main__":
-    unittest.main()
+if __name__ == '__main__':
+    unittest.main()