Applied "Introduction of ItemSource interface" patch

README.md

@@ -59,7 +59,12 @@ cellxgene-gateway
 Here's what the environment variables mean:
 
 * `CELLXGENE_LOCATION` - the location of the cellxgene executable, e.g. `~/anaconda2/envs/cellxgene/bin/cellxgene`
+
+At least one of the following is required:
 * `CELLXGENE_DATA` - a directory that can contain subdirectories with `.h5ad` data files, *without* trailing slash, e.g. `/mnt/cellxgene_data`
+* `CELLXGENE_BUCKET` - an s3 bucket that can contain keys with `.h5ad` data files, e.g. `my-cellxgene-data-bucket`
+Cellxgene Gateway is designed to make it easy to add additional data sources, please see the source code for gateway.py and the ItemSource interface in items/item_source.py
+
 Optional environment variables:
 * `CELLXGENE_ARGS` - catch-all variable that can be used to pass additional command line args to cellxgene server
 * `EXTERNAL_HOST` - the hostname and port from the perspective of the web browser, typically `localhost:5005` if running locally. Defaults to "localhost:{GATEWAY_PORT}"
@@ -67,7 +72,6 @@ Optional environment variables:
 * `GATEWAY_IP` - ip addess of instance gateway is running on, mostly used to display SSH instructions. Defaults to `socket.gethostbyname(socket.gethostname())`
 * `GATEWAY_PORT` - local port that the gateway should bind to, defaults to 5005
 * `GATEWAY_EXTRA_SCRIPTS` - JSON array of script paths, will be embedded into each page and forwarded with `--scripts` to cellxgene server
-* `GATEWAY_ENABLE_UPLOAD` - Set to `true` or `1` to enable HTTP uploads. This is not recommended for a public server.
 * `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations. 
 * `GATEWAY_ENABLE_BACKED_MODE` - Set to `true` or to `1` to load AnnData in file-backed mode. This saves memory and speeds up launch time but may reduce overall performance.  
 
@@ -126,9 +130,8 @@ For convenience, the code repo includes a `run.sh.example` shell script to run t
 pip install isort flake8 black
 
 ```bash
-isort -rc .
+isort -rc . # rc means recursive, and was deprecated in dev version of isort
-flake8 .
+black .
-black -l 79 .
 ```
 
 # Getting Help

cellxgene_gateway/backend_cache.py

@@ -14,8 +14,10 @@ from flask_api import status
 
 from cellxgene_gateway import env
 from cellxgene_gateway.cache_entry import CacheEntry, CacheEntryStatus
+from cellxgene_gateway.cache_key import CacheKey
 from cellxgene_gateway.cellxgene_exception import CellxgeneException
 from cellxgene_gateway.subprocess_backend import SubprocessBackend
+from typing import List
 
 process_backend = SubprocessBackend()
 
@@ -35,13 +37,13 @@ class BackendCache:
         contents = self.entry_list
         return [c.port for c in contents]
 
-    def check_entry(self, key):
+    def check_path(self, source, path):
         contents = self.entry_list
         matches = [
             c
             for c in contents
-            if c.key.dataset == key.dataset
+            if c.key.source.name == source.name
-            and c.key.annotation_file == key.annotation_file
+            and path.startswith(c.key.descriptor)
             and c.status != CacheEntryStatus.terminated
         ]
 
@@ -52,10 +54,28 @@ class BackendCache:
         else:
             raise CellxgeneException(
                 status.HTTP_500_INTERNAL_SERVER_ERROR,
-                "Found " + str(len(matches)) + " for " + dataset,
+                "Found " + str(len(matches)) + " for " + path,
             )
 
-    def create_entry(self, key, scripts):
+    def check_entry(self, key):
+        contents = self.entry_list
+        matches = [
+            c
+            for c in contents
+            if c.key.equals(key) and c.status != CacheEntryStatus.terminated
+        ]
+
+        if len(matches) == 0:
+            return None
+        elif len(matches) == 1:
+            return matches[0]
+        else:
+            raise CellxgeneException(
+                status.HTTP_500_INTERNAL_SERVER_ERROR,
+                "Found " + str(len(matches)) + " for " + key.dataset,
+            )
+
+    def create_entry(self, key: CacheKey, scripts: List[str]):
         port = 8000
         existing_ports = self.get_ports()
 

cellxgene_gateway/cache_entry.py

@@ -8,9 +8,10 @@
 # the specific language governing permissions and limitations under the License.
 import datetime
 import logging
+import urllib.parse
+from enum import Enum
 
 import psutil
-from enum import Enum
 from flask import make_response, render_template, request
 from requests import get, post, put
 import re
@@ -69,6 +70,10 @@ class CacheEntry:
             None,
         )
 
+    @property
+    def source_name(self):
+        return self.key.source_name
+
     def set_loaded(self, pid):
         self.pid = pid
         self.status = CacheEntryStatus.loaded
@@ -98,9 +103,13 @@ class CacheEntry:
             for child in children:
                 child.terminate()
             psutil.wait_procs(children, callback=on_terminate)
-            terminated.append(p.pid)
+            # the parent process may automatically die once its children have --
-            p.terminate()
+            try:
-            psutil.wait_procs([p], callback=on_terminate)
+                p.terminate()
+                psutil.wait_procs([p], callback=on_terminate)
+            except psutil.NoSuchProcess:
+                pass
+
             logging.getLogger("cellxgene_gateway").info(
                 f"terminated {terminated}"
             )
@@ -120,15 +129,19 @@ class CacheEntry:
         return gateway_content
 
     def gateway_basepath(self):
-        return f"{env.external_protocol}://{env.external_host}/view/{self.key.pathpart}/"
+        source_path = (
+            f"/source/{urllib.parse.quote_plus(self.source_name)}"
+            if self.source_name
+            else ""
+        )
+        return f"{env.external_protocol}://{env.external_host}{source_path}/view/{self.key.descriptor}/"
 
     def cellxgene_basepath(self):
         return f"http://127.0.0.1:{self.port}"
 
     def serve_content(self, path):
         gateway_basepath = self.gateway_basepath()
-        subpath = path[len(self.key.pathpart) :]  # noqa: E203
+        subpath = path[len(self.key.descriptor) :]  # noqa: E203
-
         if len(subpath) == 0:
             r = make_response(f"Redirect to {gateway_basepath}\n", 301)
             r.headers["location"] = gateway_basepath + querystring()
@@ -201,5 +214,4 @@ class CacheEntry:
             cellxgene_response.status_code,
             resp_headers,
         )
-
         return gateway_response

cellxgene_gateway/cache_key.py

@@ -9,15 +9,62 @@
 
 # There are three kinds of CacheKey:
 # 1) somedir/dataset.h5ad: a dataset
-#    in this case, pathpart == dataset == 'somedir/dataset.h5ad'
+#    in this case, descriptor == dataset == 'somedir/dataset.h5ad'
-# 2) somedir/dataset_annotations/my_annotations.csv : an actual annotaitons file.
+# 2) somedir/dataset_annotations/my_annotations.csv : an actual annotations file.
-#    in this case, pathpart == 'dataset_annotations/my_annotations.csv', dataset == 'somedir/dataset.h5ad'
+#    in this case, descriptor == 'somedir/dataset_annotations/my_annotations.csv', dataset == 'somedir/dataset.h5ad'
 # 3) somedir/dataset_annotations: an annotation directory. The corresponding h5ad must exist, but the directory may not.
-#    in this case, pathpart == 'dataset_annotations', dataset == 'somedir/dataset.h5ad'
+#    in this case, descriptor == 'somedir/dataset_annotations', dataset == 'somedir/dataset.h5ad'
+
+from cellxgene_gateway.items.item import Item
+from cellxgene_gateway.items.item_source import ItemSource, LookupResult
 
 
 class CacheKey:
-    def __init__(self, pathpart, dataset, annotation_file):
+    @property
-        self.pathpart = pathpart
+    def descriptor(self):
-        self.dataset = dataset
+        if self.annotation_item is None:
-        self.annotation_file = annotation_file
+            return self.h5ad_item.descriptor
+        else:
+            return self.annotation_item.descriptor
+
+    @property
+    def file_path(self):
+        return self.source.get_local_path(self.h5ad_item)
+
+    @property
+    def annotation_file_path(self):
+        if self.annotation_item is None:
+            return None
+        else:
+            return self.source.get_local_path(self.annotation_item)
+
+    @property
+    def source_name(self):
+        return self.source.name
+
+    @property
+    def annotation_descriptor(self):
+        if self.annotation_item is None:
+            return None
+        else:
+            return self.annotation_item.descriptor
+
+    def equals(self, other):
+        return (
+            (self.source.name == other.source.name)
+            and (self.h5ad_item.descriptor == other.h5ad_item.descriptor)
+            and (self.annotation_descriptor == other.annotation_descriptor)
+        )
+
+    def __init__(
+        self, h5ad_item: Item, source: ItemSource, annotation_item: Item = None
+    ):
+        assert h5ad_item is not None
+        assert source is not None
+        self.h5ad_item = h5ad_item
+        self.annotation_item = annotation_item
+        self.source = source
+
+    @classmethod
+    def for_lookup(cls, source: ItemSource, lookup: LookupResult):
+        return CacheKey(lookup.h5ad_item, source, lookup.annotation_item)

cellxgene_gateway/dir_util.py

@@ -53,10 +53,11 @@ def create_dir(parent_path, dir_name):
 
 
 annotations_suffix = "_annotations"
+h5ad_suffix = ".h5ad"
 
 
 def make_h5ad(el):
-    return el[: -len(annotations_suffix)] + ".h5ad"
+    return el[: -len(annotations_suffix)] + h5ad_suffix
 
 
 def make_annotations(el):

cellxgene_gateway/env.py

@@ -22,13 +22,9 @@ external_host = os.environ.get(
 external_protocol = os.environ.get(
     "EXTERNAL_PROTOCOL", os.environ.get("GATEWAY_PROTOCOL", "http")
 )
-ip = os.environ.get("GATEWAY_IP", "127.0.0.1")
+ip = os.environ.get("GATEWAY_IP")
 extra_scripts = os.environ.get("GATEWAY_EXTRA_SCRIPTS")
 ttl = os.environ.get("GATEWAY_TTL")
-enable_upload = os.environ.get("GATEWAY_ENABLE_UPLOAD", "").lower() in [
-    "true",
-    "1",
-]
 enable_annotations = os.environ.get(
     "GATEWAY_ENABLE_ANNOTATIONS", ""
 ).lower() in [
@@ -44,7 +40,6 @@ enable_backed_mode = os.environ.get(
 
 env_vars = {
     "CELLXGENE_LOCATION": cellxgene_location,
-    "CELLXGENE_DATA": cellxgene_data,
 }
 
 optional_env_vars = {
@@ -54,10 +49,10 @@ optional_env_vars = {
     "GATEWAY_PORT": gateway_port,
     "GATEWAY_EXTRA_SCRIPTS": extra_scripts,
     "GATEWAY_TTL": ttl,
-    "GATEWAY_ENABLE_UPLOAD": enable_upload,
     "GATEWAY_ENABLE_ANNOTATIONS": enable_annotations,
     "GATEWAY_ENABLE_BACKED_MODE": enable_backed_mode,
     "CELLXGENE_ARGS": cellxgene_args,
+    "CELLXGENE_DATA": cellxgene_data,
 }
 
 

cellxgene_gateway/filecrawl.py

@@ -8,113 +8,61 @@
 # the specific language governing permissions and limitations under the License.
 
 import os
-from cellxgene_gateway import env
+import urllib.parse
-from cellxgene_gateway.dir_util import (
-    make_h5ad,
-    make_annotations,
-    annotations_suffix,
-)
 
 
-def recurse_dir(path):
+def render_annotations(item, item_source):
-    if not os.path.exists(path):
+    subpath = f"/source/{urllib.parse.quote_plus(item_source.name)}/view/"
-        raise CellxgeneException(
+    new_annotation = f"<a class='new' href='{subpath}{item_source.get_annotations_subpath(item)}'>new</a>"
-            "The given path does not exist.", status.HTTP_400_BAD_REQUEST
+    annotations = (
-        )
+        ", ".join(
-
-    all_entries = sorted(os.listdir(path))
-
-    def is_h5ad(el):
-        return el.endswith(".h5ad") and os.path.isfile(os.path.join(path, el))
-
-    h5ad_entries = [x for x in all_entries if is_h5ad(x)]
-    annotation_dir_entries = [
-        x
-        for x in all_entries
-        if x.endswith(annotations_suffix) and make_h5ad(x) in h5ad_entries
-    ]
-
-    def list_annotations(el):
-        full_path = os.path.join(path, el)
-        if not os.path.isdir(full_path):
-            entries = []
-        else:
-            entries = [
-                {
-                    "name": x[:-13]
-                    if (len(x) > 13 and x[-13] in ["-", "_"])
-                    else (x[:-4] if x.endswith(".csv") else x),
-                    "path": os.path.join(full_path, x).replace(
-                        env.cellxgene_data, ""
-                    ),
-                }
-                for x in sorted(os.listdir(full_path))
-                if x.endswith(".csv")
-                and os.path.isfile(os.path.join(full_path, x))
-            ]
-        return [
-            {
-                "name": "new",
-                "class": "new",
-                "path": full_path.replace(env.cellxgene_data, ""),
-            }
-        ] + entries
-
-    def make_entry(el):
-        full_path = os.path.join(path, el)
-        if el in h5ad_entries:
-            return {
-                "path": full_path.replace(env.cellxgene_data, ""),
-                "name": el,
-                "type": "file",
-                "annotations": list_annotations(make_annotations(el)),
-            }
-        elif os.path.isdir(full_path) and el not in annotation_dir_entries:
-            return {
-                "path": full_path.replace(env.cellxgene_data, ""),
-                "name": el,
-                "type": "directory",
-                "children": recurse_dir(full_path),
-            }
-        else:
-            return {
-                "path": full_path,
-                "name": el,
-                "type": "neither",
-            }
-
-    return [make_entry(x) for x in all_entries]
-
-
-def render_entries(entries):
-    return "<ul>" + "\n".join([render_entry(e) for e in entries]) + "</ul>"
-
-
-def get_url(entry):
-    return f"/view/{ entry['path'].lstrip('/') }"
-
-
-def get_class(entry):
-    return f" class='{entry['class']}'" if "class" in entry else ""
-
-
-def render_annotations(entry):
-    if len(entry["annotations"]) > 0:
-        return " | annotations: " + ", ".join(
             [
-                f"<a href='{get_url(a)}'{get_class(a)}>{a['name']}</a>"
+                f"<a href='{subpath}{a.descriptor}/'>{a.name}</a>"
-                for a in entry["annotations"]
+                for a in item.annotations
             ]
         )
-    else:
+        + ", "
-        return ""
+        if item.annotations
+        else ""
+    )
+
+    return " | annotations: " + annotations + new_annotation
 
 
-def render_entry(entry):
+def render_item(item, item_source):
-    if entry["type"] == "file":
+    url = f"/source/{urllib.parse.quote_plus(item_source.name)}/view/{item.descriptor}/"
-        return f"<li> <a href='{ get_url(entry) }'>{entry['name']}</a> {render_annotations(entry)}</li>"
+    item_string = f"<li> <a href='{ url }'>{item.name}</a> {render_annotations(item, item_source)}</li>"
-    elif entry["type"] == "directory":
+    return item_string
-        url = f"/filecrawl/{entry['path'].lstrip('/')}"
+
-        return f"<li><a href='{url}'>{entry['name']}</a>{render_entries(entry['children'])}</li>"
+
+def render_item_tree(item_tree, item_source):
+    items = (
+        "\n".join([render_item(i, item_source) for i in item_tree.items])
+        if item_tree.items
+        else ""
+    )
+    branches = (
+        "\n".join(
+            [render_item_tree(b, item_source) for b in item_tree.branches]
+        )
+        if item_tree.branches
+        else ""
+    )
+    html = "<ul>" + items + branches + "</ul>"
+    if item_tree.descriptor:
+        descriptor = item_tree.descriptor.lstrip("/")
+        url = f"/filecrawl/{descriptor}?source={item_source.name}"
+        name = (
+            descriptor.rsplit("/")[1]
+            if descriptor.find("/") >= 0
+            else descriptor
+        )
+        return f"<li><a href='{url}'>{name}</a>{html}</li>"
     else:
-        return ""
+        return html
+
+
+def render_item_source(item_source, filter=None):
+    item_tree = item_source.list_items(filter)
+    heading = f"<h6><a href='/filecrawl.html?source={urllib.parse.quote_plus(item_source.name)}'>{item_source.name}</a></h6>"
+    return heading + render_item_tree(item_tree, item_source)

cellxgene_gateway/gateway.py

@@ -6,12 +6,11 @@
 # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
-
+# import BaseHTTPServer
 import json
 import logging
-
-# import BaseHTTPServer
 import os
+import urllib.parse
 from threading import Lock, Thread
 
 from flask import (
@@ -32,14 +31,17 @@ from cellxgene_gateway.cache_entry import CacheEntryStatus
 from cellxgene_gateway.cellxgene_exception import CellxgeneException
 from cellxgene_gateway.dir_util import create_dir, is_subdir
 from cellxgene_gateway.extra_scripts import get_extra_scripts
-from cellxgene_gateway.filecrawl import recurse_dir, render_entries
+from cellxgene_gateway.filecrawl import render_item_source
-from cellxgene_gateway.path_util import get_key
 from cellxgene_gateway.process_exception import ProcessException
 from cellxgene_gateway.prune_process_cache import PruneProcessCache
 from cellxgene_gateway.util import current_time_stamp
+from cellxgene_gateway.cache_key import CacheKey
 
 app = Flask(__name__)
 
+item_sources = []
+default_item_source = None
+
 
 def _force_https(app):
     def wrapper(environ, start_response):
@@ -88,8 +90,8 @@ def handle_invalid_process(error):
             http_status=error.http_status,
             stdout=error.stdout,
             stderr=error.stderr,
-            dataset=error.key.dataset,
+            dataset=error.key.h5ad_item.descriptor,
-            annotation_file=error.key.annotation_file,
+            annotation_file=error.key.annotation_descriptor,
         ),
         error.http_status,
     )
@@ -106,84 +108,40 @@ def favicon():
 
 @app.route("/")
 def index():
-    users = [
-        name
-        for name in os.listdir(env.cellxgene_data)
-        if os.path.isdir(os.path.join(env.cellxgene_data, name))
-    ]
     return render_template(
         "index.html",
         ip=env.ip,
         cellxgene_data=env.cellxgene_data,
         extra_scripts=get_extra_scripts(),
-        users=users,
-        enable_upload=env.enable_upload,
     )
 
 
-def make_user():
+@app.route("/filecrawl.html")
-    dir_name = request.form["directory"]
+@app.route("/filecrawl/<path:path>")
-
+def filecrawl(path=None):
-    create_dir(env.cellxgene_data, dir_name)
+    source_name = request.args.get("source")
-
+    sources = (
-    return redirect(location, code=302)
+        filter(
-
+            lambda x: x.name == urllib.parse.unquote_plus(source_name),
-
+            item_sources,
-def make_subdir():
-    parent_path = os.path.join(env.cellxgene_data, request.form["usernames"])
-    dir_name = request.form["directory"]
-
-    create_dir(parent_path, dir_name)
-
-    return redirect(location, code=302)
-
-
-def upload_file():
-    upload_dir = request.form["path"]
-
-    full_upload_path = os.path.join(env.cellxgene_data, upload_dir)
-    if is_subdir(full_upload_path, env.cellxgene_data) and os.path.isdir(
-        full_upload_path
-    ):
-        if request.method == "POST":
-            if "file" in request.files:
-                f = request.files["file"]
-                if f and f.filename.endswith(".h5ad"):
-                    f.save(
-                        os.path.join(
-                            full_upload_path, secure_filename(f.filename)
-                        )
-                    )
-                    return redirect("/filecrawl.html", code=302)
-                else:
-                    raise CellxgeneException(
-                        "Uploaded file must be in anndata (.h5ad) format.",
-                        status.HTTP_400_BAD_REQUEST,
-                    )
-            else:
-                raise CellxgeneException(
-                    "A file must be chosen to upload.",
-                    status.HTTP_400_BAD_REQUEST,
-                )
-    else:
-        raise CellxgeneException(
-            "Invalid directory.", status.HTTP_400_BAD_REQUEST
         )
-
+        if source_name
-    return redirect(location, code=302)
+        else item_sources
-
-
-if env.enable_upload:
-    app.add_url_rule("/make_user", "make_user", make_user, methods=["POST"])
-    app.add_url_rule(
-        "/make_subdir", "make_subdir", make_subdir, methods=["POST"]
     )
-    app.add_url_rule(
+    # loop all data sources --
-        "/upload_file", "upload_file", upload_file, methods=["POST"]
+    rendered_sources = [
+        render_item_source(item_source, path) for item_source in sources
+    ]  # will we need to make this async in the page???
+    rendered_html = "\n".join(rendered_sources)
+
+    resp = make_response(
+        render_template(
+            "filecrawl.html",
+            extra_scripts=get_extra_scripts(),
+            rendered_html=rendered_html,
+            path=path,
+        )
     )
-
-
-def set_no_cache(resp):
     resp.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
     resp.headers["Pragma"] = "no-cache"
     resp.headers["Expires"] = "0"
@@ -191,52 +149,44 @@ def set_no_cache(resp):
     return resp
 
 
-@app.route("/filecrawl.html")
-def filecrawl():
-    entries = recurse_dir(env.cellxgene_data)
-    rendered_html = render_entries(entries)
-    resp = make_response(
-        render_template(
-            "filecrawl.html",
-            extra_scripts=get_extra_scripts(),
-            rendered_html=rendered_html,
-        )
-    )
-    return set_no_cache(resp)
-
-
-@app.route("/filecrawl/<path:path>")
-def do_filecrawl(path):
-    filecrawl_path = os.path.join(env.cellxgene_data, path)
-    if not os.path.isdir(filecrawl_path):
-        raise CellxgeneException(
-            "Path is not directory: " + filecrawl_path,
-            status.HTTP_400_BAD_REQUEST,
-        )
-    entries = recurse_dir(filecrawl_path)
-    rendered_html = render_entries(entries)
-    return render_template(
-        "filecrawl.html",
-        extra_scripts=get_extra_scripts(),
-        rendered_html=rendered_html,
-        path=path,
-    )
-
-
 entry_lock = Lock()
 
 
+def matching_source(source_name):
+    if source_name is None:
+        source_name = default_item_source.name
+    matching = [i for i in item_sources if i.name == source_name]
+    if len(matching) != 1:
+        raise Exception(f"Could not find matching item source {source_name}")
+    source = matching[0]
+    return source
+
+
+@app.route(
+    "/source/<path:source_name>/view/<path:path>",
+    methods=["GET", "PUT", "POST"],
+)
 @app.route("/view/<path:path>", methods=["GET", "PUT", "POST"])
-def do_view(path):
+def do_view(path, source_name=None):
-    key = get_key(path)
+    source = matching_source(source_name)
-    print(
+    match = cache.check_path(source, path)
-        f"view path={path}, dataset={key.dataset}, annotation_file= {key.annotation_file}, key={key.pathpart}"
+
-    )
+    if match is None:
-    with entry_lock:
+        lookup = source.lookup(path)
-        match = cache.check_entry(key)
+        if lookup is None:
-        if match is None:
+            raise CellxgeneException(
-            uascripts = get_extra_scripts()
+                f"Could not find item for path {path} in source {source.name}",
-            match = cache.create_entry(key, uascripts)
+                404,
+            )
+        key = CacheKey.for_lookup(source, lookup)
+        print(
+            f"view path={path}, source_name={source_name}, dataset={key.file_path}, annotation_file= {key.annotation_file_path}, key={key.descriptor}, source={key.source_name}"
+        )
+        with entry_lock:
+            match = cache.check_entry(key)
+            if match is None:
+                uascripts = get_extra_scripts()
+                match = cache.create_entry(key, uascripts)
 
     match.timestamp = current_time_stamp()
 
@@ -275,7 +225,9 @@ def do_GET_status_json():
 
 @app.route("/relaunch/<path:path>", methods=["GET"])
 def do_relaunch(path):
-    key = get_key(path)
+    source_name = request.args.get("source") or default_item_source.name
+    source = matching_source(source_name)
+    key = CacheKey.for_lookup(source, source.lookup(path))
     match = cache.check_entry(key)
     if not match is None:
         match.terminate()
@@ -288,25 +240,24 @@ def do_relaunch(path):
 
 @app.route("/terminate/<path:path>", methods=["GET"])
 def do_terminate(path):
-    key = get_key(path)
+    source_name = request.args.get("source_name") or default_item_source.name
+    source = matching_source(source_name)
+    key = CacheKey.for_lookup(source, source.lookup(path))
     match = cache.check_entry(key)
     if not match is None:
         match.terminate()
     return redirect(url_for("do_GET_status"), code=302)
 
 
-@app.route("/metadata/ip_address", methods=["GET"])
+def launch():
-def ip_address():
-    resp = make_response(env.ip)
-    return set_no_cache(resp)
-
-
-def main():
-    logging.basicConfig(
-        level=logging.INFO,
-        format="%(asctime)s:%(name)s:%(levelname)s:%(message)s",
-    )
     env.validate()
+    if not item_sources or not len(item_sources):
+        raise Exception("No data sources specified for Cellxgene Gateway")
+
+    global default_item_source
+    if default_item_source is None:
+        default_item_source = item_sources[0]
+
     pruner = PruneProcessCache(cache)
 
     background_thread = Thread(target=pruner)
@@ -316,5 +267,29 @@ def main():
     app.run(host="0.0.0.0", port=env.gateway_port, debug=False)
 
 
+def main():
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s:%(name)s:%(levelname)s:%(message)s",
+    )
+    cellxgene_data = os.environ.get("CELLXGENE_DATA", None)
+    cellxgene_bucket = os.environ.get("CELLXGENE_BUCKET", None)
+
+    if cellxgene_bucket is not None:
+        from cellxgene_gateway.items.s3.s3item_source import S3ItemSource
+
+        item_sources.append(S3ItemSource(cellxgene_bucket, name="s3"))
+        default_item_source = "s3"
+    if cellxgene_data is not None:
+        from cellxgene_gateway.items.file.fileitem_source import FileItemSource
+
+        item_sources.append(FileItemSource(cellxgene_data, name="local"))
+        default_item_source = "local"
+    if len(item_sources) == 0:
+        raise Exception("Please specify CELLXGENE_DATA or CELLXGENE_BUCKET")
+
+    launch()
+
+
 if __name__ == "__main__":
     main()

cellxgene_gateway/items/file/fileitem.py

@@ -0,0 +1,27 @@
+# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
+# under the Apache License, Version 2.0 (the "License"); you may not use
+# this file except in compliance with the License. You may obtain a copy
+# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
+# required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+# the specific language governing permissions and limitations under the License.
+
+import os
+
+from cellxgene_gateway.items.item import Item
+
+
+class FileItem(Item):
+    """e.g. FileItem(subpath = subpath, name = filename, type = ItemType.h5ad)
+
+    The Item superclass expects a 'name' and 'type'.
+    """
+
+    def __init__(self, subpath: str, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.subpath = subpath
+
+    @property
+    def descriptor(self) -> str:
+        return os.path.join(self.subpath, self.name).strip("/")

cellxgene_gateway/items/file/fileitem_source.py

@@ -0,0 +1,185 @@
+# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
+# under the Apache License, Version 2.0 (the "License"); you may not use
+# this file except in compliance with the License. You may obtain a copy
+# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
+# required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+# the specific language governing permissions and limitations under the License.
+
+import os
+from typing import List
+
+from cellxgene_gateway import dir_util
+from cellxgene_gateway.items.file.fileitem import FileItem
+from cellxgene_gateway.items.item import ItemTree, ItemType
+from cellxgene_gateway.items.item_source import ItemSource, LookupResult
+
+
+class FileItemSource(ItemSource):
+    def __init__(
+        self,
+        base_path,
+        name=None,
+        h5ad_suffix=dir_util.h5ad_suffix,
+        annotation_dir_suffix=dir_util.annotations_suffix,
+        annotation_file_suffix=".csv",
+    ):
+        self._name = name
+        self.base_path = base_path
+        self.h5ad_suffix = h5ad_suffix
+        self.annotation_dir_suffix = annotation_dir_suffix
+        self.annotation_file_suffix = annotation_file_suffix
+
+    @property
+    def name(self):
+        return self._name or f"Files:{self.base_path}"
+
+    def is_h5ad_file(self, path: str) -> bool:
+        return path.endswith(self.h5ad_suffix) and os.path.isfile(path)
+
+    def convert_annotation_path_to_h5ad(self, path):
+        return path[: -len(self.annotation_dir_suffix)] + self.h5ad_suffix
+
+    def convert_h5ad_path_to_annotation(self, path):
+        return path[: -len(self.h5ad_suffix)] + self.annotation_dir_suffix
+
+    def get_local_path(self, item: FileItem) -> str:
+        return os.path.join(self.base_path, item.descriptor)
+
+    def get_annotations_subpath(self, item) -> str:
+        return self.convert_h5ad_path_to_annotation(item.descriptor)
+
+    def list_items(self, filter: str = None) -> ItemTree:
+        item_tree = self.scan_directory()
+
+        """def get_items(dir):
+            if dir.branches:
+                return [*dir.items, *[item for subdir in dir.branches for item in get_items(subdir)]]
+            else:
+                return dir.items
+
+        return get_items(self.item_tree)"""
+
+        return item_tree
+
+    def scan_directory(self, subpath="") -> dict:
+        base_path = os.path.join(self.base_path, subpath)
+
+        if not os.path.exists(base_path):
+            raise Exception(
+                f"Path for local files '{base_path}' does not exist."
+            )
+
+        filepath_map = dict(
+            (filepath, os.path.join(base_path, filepath))
+            for filepath in sorted(os.listdir(base_path))
+        )
+
+        def is_annotation_dir(dir):
+            return (
+                dir.endswith(self.annotation_dir_suffix)
+                and self.convert_annotation_path_to_h5ad(dir) in h5ad_paths
+            )
+
+        h5ad_paths = [
+            filepath
+            for filepath, full_path in filepath_map.items()
+            if self.is_h5ad_file(full_path)
+        ]
+
+        subdirs = [
+            filepath
+            for filepath, full_path in filepath_map.items()
+            if os.path.isdir(full_path) and not is_annotation_dir(filepath)
+        ]
+
+        items = [
+            self.make_fileitem_from_path(filename, subpath)
+            for filename in h5ad_paths
+        ]
+        branches = None
+        if len(subdirs) > 0:
+            branches = [
+                self.scan_directory(os.path.join(subpath, subdir))
+                for subdir in subdirs
+            ]
+
+        return ItemTree(subpath, items, branches)
+
+    def create_annotation(self, item: FileItem, name: str) -> FileItem:
+        annotation = self.make_fileitem_from_path(
+            name, self.get_annotations_subpath(item), is_annotation=True
+        )
+        item.annotations = (item.annotations or []).append(annotation)
+        return annotation
+
+    def update(self, item: FileItem) -> None:
+        pass
+
+    def full_path(self, p):
+        return os.path.join(self.base_path, p)
+
+    def lookup_item(self, descriptor):
+        full_path = self.full_path(descriptor)
+        if self.is_h5ad_file(full_path):
+            return self.shallowitem_from_descriptor(descriptor)
+
+    def lookup(self, indescriptor: str) -> LookupResult:
+        descriptor = indescriptor.strip("/")
+        if descriptor.endswith(self.annotation_file_suffix):
+            annotation_item = self.shallowitem_from_descriptor(
+                descriptor, True
+            )
+            h5ad_descriptor = self.convert_annotation_path_to_h5ad(
+                annotation_item.subpath
+            )
+            item = self.lookup_item(h5ad_descriptor)
+            if item is not None:
+                return LookupResult(item, annotation_item)
+        else:
+            item = self.lookup_item(descriptor)
+            if item is not None:
+                return LookupResult(item)
+
+    def shallowitem_from_descriptor(self, descriptor, is_annotation=False):
+        filename = os.path.basename(descriptor)
+        subpath = os.path.dirname(descriptor)
+        return self.make_fileitem_from_path(
+            filename,
+            subpath,
+            is_annotation,
+            True,
+        )
+
+    def make_fileitem_from_path(
+        self, filename, subpath, is_annotation=False, is_shallow=False
+    ) -> FileItem:
+        item = FileItem(
+            subpath=subpath,
+            name=filename,
+            type=ItemType.annotation if is_annotation else ItemType.h5ad,
+        )
+
+        if not is_annotation and not is_shallow:
+            annotations = self.make_annotations_for_fileitem(item)
+            item.annotations = annotations
+
+        return item
+
+    def make_annotations_for_fileitem(self, item: FileItem) -> List[FileItem]:
+        annotations_subpath = self.get_annotations_subpath(item)
+        annotations_fullpath = self.full_path(annotations_subpath)
+        if os.path.isdir(annotations_fullpath):
+            return [
+                self.make_fileitem_from_path(
+                    annotation, annotations_subpath, True
+                )
+                for annotation in sorted(os.listdir(annotations_fullpath))
+                if annotation.endswith(self.annotation_file_suffix)
+                and os.path.isfile(
+                    os.path.join(annotations_fullpath, annotation)
+                )
+            ]
+        else:
+            return None

cellxgene_gateway/items/item.py

@@ -0,0 +1,43 @@
+# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
+# under the Apache License, Version 2.0 (the "License"); you may not use
+# this file except in compliance with the License. You may obtain a copy
+# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
+# required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+# the specific language governing permissions and limitations under the License.
+
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import List
+
+
+class ItemType(Enum):
+    annotation = "annotation"
+    h5ad = "h5ad"
+
+
+class Item(ABC):
+    def __init__(
+        self, name: str, type: ItemType, annotations: List["Item"] = None
+    ):
+        self.name = name
+        self.type = type
+        self.annotations = annotations
+
+    @property
+    @abstractmethod
+    def descriptor(self):
+        raise Exception('"descriptor" not implemented')
+
+
+class ItemTree:
+    def __init__(
+        self,
+        descriptor: str,
+        items: List[Item] = None,
+        branches: List["ItemTree"] = None,
+    ):
+        self.descriptor = descriptor
+        self.items = items
+        self.branches = branches

cellxgene_gateway/items/item_source.py

@@ -0,0 +1,50 @@
+# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
+# under the Apache License, Version 2.0 (the "License"); you may not use
+# this file except in compliance with the License. You may obtain a copy
+# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
+# required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+# the specific language governing permissions and limitations under the License.
+
+from abc import ABC, abstractmethod
+from typing import List
+
+from cellxgene_gateway.items.item import Item
+
+
+class LookupResult:
+    def __init__(self, h5ad_item: Item, annotation_item: Item = None):
+        self.h5ad_item = h5ad_item
+        self.annotation_item = annotation_item
+
+
+class ItemSource(ABC):
+    @abstractmethod
+    def list_items(self, filter: str = None) -> List[Item]:
+        raise Exception('"list_items" unimplemented')
+
+    @abstractmethod
+    def get_local_path(self, item: Item) -> str:
+        raise Exception('"local_path" unimplemented')
+
+    @abstractmethod
+    def get_annotations_subpath(self, item) -> str:
+        raise Exception('"annotations_path" unimplemented')
+
+    @abstractmethod
+    def create_annotation(self, item: Item, name: str) -> Item:
+        raise Exception('"annotation" unimplemented')
+
+    @abstractmethod
+    def update(self, item: Item) -> None:
+        raise Exception('"update" unimplemented')
+
+    @abstractmethod
+    def lookup(self, descriptor: str) -> LookupResult:
+        raise Exception('"lookup" unimplemented')
+
+    @property
+    @abstractmethod
+    def name(self):
+        pass

cellxgene_gateway/items/s3/s3item.py

@@ -0,0 +1,27 @@
+# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
+# under the Apache License, Version 2.0 (the "License"); you may not use
+# this file except in compliance with the License. You may obtain a copy
+# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
+# required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+# the specific language governing permissions and limitations under the License.
+
+import os
+
+from cellxgene_gateway.items.item import Item
+
+
+class S3Item(Item):
+    """e.g. FileItem(subpath = subpath, name = filename, type = ItemType.h5ad)
+
+    The Item superclass expects a 'name' and 'type'.
+    """
+
+    def __init__(self, s3key: str, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.s3key = s3key
+
+    @property
+    def descriptor(self) -> str:
+        return self.s3key

cellxgene_gateway/items/s3/s3item_source.py

@@ -0,0 +1,171 @@
+# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
+# under the Apache License, Version 2.0 (the "License"); you may not use
+# this file except in compliance with the License. You may obtain a copy
+# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
+# required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+# the specific language governing permissions and limitations under the License.
+
+from typing import List
+from os.path import join, dirname, basename
+from cellxgene_gateway import dir_util
+import s3fs
+from cellxgene_gateway.items.s3.s3item import S3Item
+from cellxgene_gateway.items.item import ItemTree, ItemType
+from cellxgene_gateway.items.item_source import ItemSource, LookupResult
+
+
+class S3ItemSource(ItemSource):
+    def __init__(
+        self,
+        bucket,
+        name=None,
+        h5ad_suffix=dir_util.h5ad_suffix,
+        annotation_dir_suffix=dir_util.annotations_suffix,
+        annotation_file_suffix=".csv",
+    ):
+        self._name = name
+        self.s3 = s3fs.S3FileSystem()
+        self.bucket = bucket
+        self.h5ad_suffix = h5ad_suffix
+        self.annotation_dir_suffix = annotation_dir_suffix
+        self.annotation_file_suffix = annotation_file_suffix
+
+    def url(self, path):
+        return "s3://" + join(self.bucket, path)
+
+    @property
+    def name(self):
+        return self._name or f"Items:{self.url('')}"
+
+    def is_h5ad_url(self, s3url: str) -> bool:
+        return s3url.endswith(self.h5ad_suffix) and self.s3.exists(s3url)
+
+    def convert_annotation_key_to_h5ad(self, s3key):
+        return s3key[: -len(self.annotation_dir_suffix)] + self.h5ad_suffix
+
+    def convert_h5ad_key_to_annotation(self, s3key):
+        return s3key[: -len(self.h5ad_suffix)] + self.annotation_dir_suffix
+
+    def get_local_path(self, item: S3Item) -> str:
+        return self.url(item.descriptor)
+
+    def get_annotations_subpath(self, item) -> str:
+        return self.convert_h5ad_key_to_annotation(item.descriptor)
+
+    def list_items(self, filter: str = None) -> ItemTree:
+        item_tree = self.scan_directory()
+        return item_tree
+
+    def scan_directory(self, subpath="") -> dict:
+        url = self.url(subpath)
+
+        if not self.s3.exists(url):
+            raise Exception(f"S3 url '{url}' does not exist.")
+
+        s3key_map = dict(
+            (filepath[len(self.bucket) :], "s3://" + filepath)
+            for filepath in sorted(self.s3.ls(url))
+        )
+
+        def is_annotation_dir(dir_s3key):
+            return (
+                dir_s3key.endswith(self.annotation_dir_suffix)
+                and self.convert_annotation_key_to_h5ad(dir_s3key)
+                in h5ad_paths
+            )
+
+        h5ad_paths = [
+            filepath
+            for filepath, item_url in s3key_map.items()
+            if self.is_h5ad_url(item_url)
+        ]
+
+        subdirs = [
+            filepath
+            for filepath, item_url in s3key_map.items()
+            if self.s3.isdir(item_url) and not is_annotation_dir(filepath)
+        ]
+
+        items = [
+            self.make_s3item_from_key(filename, join(subpath, filename))
+            for filename in h5ad_paths
+        ]
+        branches = None
+        if len(subdirs) > 0:
+            branches = [
+                self.scan_directory(join(subpath, subdir))
+                for subdir in subdirs
+            ]
+
+        return ItemTree(subpath, items, branches)
+
+    def create_annotation(self, item: S3Item, name: str) -> S3Item:
+        annotation = self.make_s3item_from_key(
+            name, self.get_annotations_subpath(item), is_annotation=True
+        )
+        item.annotations = (item.annotations or []).append(annotation)
+        return annotation
+
+    def update(self, item: S3Item) -> None:
+        pass
+
+    def lookup_item(self, descriptor):
+        full_path = self.url(descriptor)
+        if self.is_h5ad_url(full_path):
+            return self.shallowitem_from_descriptor(descriptor)
+
+    def lookup(self, indescriptor: str) -> LookupResult:
+        descriptor = indescriptor.strip("/")
+        if descriptor.endswith(self.annotation_file_suffix):
+            annotation_item = self.shallowitem_from_descriptor(
+                descriptor, True
+            )
+            if not self.s3.exists(self.url(annotation_item.s3key)):
+                with self.s3.open(self.url(annotation_item.s3key), "w") as f:
+                    f.write("")
+            h5ad_descriptor = self.convert_annotation_key_to_h5ad(
+                dirname(annotation_item.s3key)
+            )
+            item = self.shallowitem_from_descriptor(h5ad_descriptor)
+            return LookupResult(item, annotation_item)
+        else:
+            item = self.lookup_item(descriptor)
+            if item is not None:
+                return LookupResult(item)
+
+    def shallowitem_from_descriptor(self, descriptor, is_annotation=False):
+        return self.make_s3item_from_key(
+            basename(descriptor), descriptor, is_annotation, True
+        )
+
+    def make_s3item_from_key(
+        self, name, s3key, is_annotation=False, is_shallow=False
+    ) -> S3Item:
+        item = S3Item(
+            s3key=s3key,
+            name=name,
+            type=ItemType.annotation if is_annotation else ItemType.h5ad,
+        )
+
+        if not is_annotation and not is_shallow:
+            annotations = self.make_annotations_for_fileitem(item)
+            item.annotations = annotations
+
+        return item
+
+    def make_annotations_for_fileitem(self, item: S3Item) -> List[S3Item]:
+        annotations_subpath = self.get_annotations_subpath(item)
+        annotations_fullpath = self.url(annotations_subpath)
+        if self.s3.isdir(annotations_fullpath):
+            return [
+                self.make_s3item_from_key(
+                    annotation, join(annotations_subpath, annotation), True
+                )
+                for annotation in sorted(self.s3.ls(annotations_fullpath))
+                if annotation.endswith(self.annotation_file_suffix)
+                and self.s3.isfile(join(annotations_fullpath, annotation))
+            ]
+        else:
+            return None

cellxgene_gateway/path_util.py

@@ -11,43 +11,11 @@ import os
 
 from flask_api import status
 
-from cellxgene_gateway import env
 from cellxgene_gateway.cache_key import CacheKey
 from cellxgene_gateway.cellxgene_exception import CellxgeneException
 from cellxgene_gateway.dir_util import make_h5ad
 
 
-def get_key(path):
-    if path == "/" or path == "":
-        raise CellxgeneException(
-            "No matching dataset found.", status.HTTP_404_NOT_FOUND
-        )
-
-    trimmed = path[:-1] if path[-1] == "/" else path
-    try:
-        # valid paths come in three forms:
-        if trimmed.endswith(".h5ad") and data_file_exists(trimmed):
-            # 1) somedir/dataset.h5ad: a dataset
-            return CacheKey(trimmed, trimmed, None)
-        elif trimmed.endswith(".csv"):
-
-            # 2) somedir/dataset_annotations/my_annotations.csv : an actual annotations file.
-            annotations_dir = os.path.split(trimmed)[0]
-            dataset = make_h5ad(annotations_dir)
-            if data_file_exists(dataset):
-                data_dir_ensure(annotations_dir)
-                return CacheKey(trimmed, dataset, trimmed)
-        elif trimmed.endswith("_annotations") and data_dir_exists(trimmed):
-            # 3) somedir/dataset_annotations: an annotation directory. The corresponding h5ad must exist, but the directory may not.
-            dataset = make_h5ad(trimmed)
-            if data_file_exists(dataset):
-                return CacheKey(trimmed, dataset, "")
-    except CellxgeneException:
-        pass
-    split = os.path.split(trimmed)
-    return get_key(split[0])
-
-
 def validate_exists(file_path):
     if not os.path.exists(file_path):
         raise CellxgeneException(
@@ -71,37 +39,3 @@ def validate_is_dir(file_path):
             "Path is not dir: " + file_path, status.HTTP_400_BAD_REQUEST
         )
     return
-
-
-def data_file_exists(dataset):
-    file_path = os.path.join(env.cellxgene_data, dataset)
-    validate_is_file(file_path)
-    return True
-
-
-def data_dir_exists(dataset):
-    file_path = os.path.join(env.cellxgene_data, dataset)
-    validate_is_dir(file_path)
-    return True
-
-
-def data_dir_ensure(dataset):
-    file_path = os.path.join(env.cellxgene_data, dataset)
-    if not os.path.exists(file_path):
-        os.makedirs(file_path)
-
-
-def get_file_path(key):
-    dataset = key.dataset
-    file_path = os.path.join(env.cellxgene_data, dataset)
-    validate_is_file(file_path)
-    return file_path
-
-
-def get_annotation_file_path(key):
-    if key.annotation_file is None:
-        return None
-    if key.annotation_file == "":
-        return ""
-    file_path = os.path.join(env.cellxgene_data, key.annotation_file)
-    return file_path

cellxgene_gateway/static/js/annotation.js

@@ -1,4 +1,5 @@
 // neandertal javascript
+// TODO: rewrite this --
 const new_annotation_callback = (() =>{
     const suffix = `.csv`;
     return (e) => {

cellxgene_gateway/subprocess_backend.py

@@ -11,9 +11,9 @@ import logging
 import subprocess
 
 from flask_api import status
+
 from cellxgene_gateway.cache_entry import CacheEntryStatus
 from cellxgene_gateway.dir_util import make_annotations
-from cellxgene_gateway.path_util import get_annotation_file_path, get_file_path
 from cellxgene_gateway.env import (
     enable_annotations,
     enable_backed_mode,
@@ -45,8 +45,7 @@ class SubprocessBackend:
 
         cmd = (
             f"yes | {cellxgene_loc} launch {file_path}"
-            + " --port "
+            + f" --port {port}"
-            + str(port)
             + " --host 127.0.0.1"
             + extra_args
         )
@@ -57,13 +56,12 @@ class SubprocessBackend:
         return cmd
 
     def launch(self, cellxgene_loc, scripts, cache_entry):
-
         cmd = self.create_cmd(
             cellxgene_loc,
-            get_file_path(cache_entry.key),
+            cache_entry.key.file_path,
             cache_entry.port,
             scripts,
-            get_annotation_file_path(cache_entry.key),
+            cache_entry.key.annotation_file_path,
         )
         logging.getLogger("cellxgene_gateway").info(f"launching {cmd}")
         process = subprocess.Popen(

cellxgene_gateway/templates/cache_status.html

@@ -10,59 +10,68 @@
 -->
 
 <html>
+
 <head>
      <title>Cellxgene Gateway - FILE CRAWLER</title>
      <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
      <link rel="icon" type="image/png" href="{{ url_for('static', filename='nibr.ico') }}">
-	{% for script in extra_scripts %}
+     {% for script in extra_scripts %}
-	<script src="{{ script }}"></script>
+     <script src="{{ script }}"></script>
-	{% endfor %}
+     {% endfor %}
-     <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css" integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO" crossorigin="anonymous">
+     <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css"
+          integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO" crossorigin="anonymous">
 </head>
+
 <body>
      <header class="navbar navbar-expand navbar-dark flex-column flex-md-row bd-navbar">
-       <h3>Cellxgene Gateway - Cache Status</h3>
+          <h3>Cellxgene Gateway - Cache Status</h3>
      </header>
      <br>
      <table class="table">
           <thead>
-          <tr>
+               <tr>
-               <th>PID</th>
+                    <th>PID</th>
-               <th>dataset</th>
+                    <th>dataset</th>
-               <th>annotation_file</th>
+                    <th>annotation_file</th>
-               <th>port</th>
+                    <th>source</th>
-               <th>launchtime</th>
+                    <th>port</th>
-               <th>last access</th>
+                    <th>launchtime</th>
-               <th>status</th>
+                    <th>last access</th>
-               <th>message</th>
+                    <th>status</th>
-               <th>http_status</th>
+                    <th>message</th>
-               <th>actions</th>
+                    <th>http_status</th>
-          </tr>
+                    <th>actions</th>
-     </thead>
+               </tr>
-     <tbody>
+          </thead>
-	{% for entry in entry_list %}
+          <tbody>
-          <tr>
+               {% for entry in entry_list %}
-               <td>{{ entry.pid }}</td>
+               <tr>
-               <td><a href="{{ url_for('do_view', path=entry.key.pathpart) }}">{{ entry.key.dataset }}</a></td>
+                    <td>{{ entry.pid }}</td>
-               <td>{{ entry.key.annotation_file }}</td>
+                    <td><a
-               <td>{{ entry.port }}</td>
+                              href="{{ url_for('do_view', path=entry.key.descriptor, source_name=entry.key.source_name) }}">{{ entry.key.h5ad_item.descriptor }}</a>
-               <td class="timestamp">{{ entry.launchtime }}</td>
+                    </td>
-               <td class="timestamp">{{ entry.timestamp }}</td>
+                    <td>{{ entry.key.annotation_descriptor }}</td>
-               <td>{{ entry.status.name }}</td>
+                    <td>{{ entry.source_name }}</td>
-               <td>{{ entry.message }}</td>
+                    <td>{{ entry.port }}</td>
-               <td>{{ entry.http_status }}</td>
+                    <td class="timestamp">{{ entry.launchtime }}</td>
-               <td>
+                    <td class="timestamp">{{ entry.timestamp }}</td>
-                 {% if entry.status.name == 'loaded' %}
+                    <td>{{ entry.status.name }}</td>
-                   <a href="{{ url_for('do_terminate', path=entry.key.pathpart) }}"> terminate </a>
+                    <td>{{ entry.message }}</td>
-                 {% endif %}
+                    <td>{{ entry.http_status }}</td>
-               </td>
+                    <td>
-          </tr>
+                         {% if entry.status.name == 'loaded' %}
-     {% endfor %}
+                         <a
-     </tbody>
+                              href="{{ url_for('do_terminate', path=entry.key.descriptor, source_name=entry.key.source_name) }}">
+                              terminate </a>
+                         {% endif %}
+                    </td>
+               </tr>
+               {% endfor %}
+          </tbody>
      </table>
      <script>
           $(() => {
-               $(".timestamp").each(function(){
+               $(".timestamp").each(function () {
                     const el = $(this);
                     const ts = el.text();
                     const dt = new Date(parseInt(ts * 1000));
@@ -71,4 +80,5 @@
           })
      </script>
 </body>
-</html>
+
+</html>

cellxgene_gateway/templates/index.html

@@ -44,47 +44,6 @@
 			<u>Cache Status: view status of launched cellxgene servers.</u></a>
 	</div>
 	
-	{% if enable_upload %}
-	<br>
-	<h1 style="padding-left:35px">
-		How To Upload Data via HTTP:
-	</h1>
-	<ol style="padding-left:85px;">
-		<li>
-			Create a folder for your Username: 
-		</li>
-		<br>
-		<form action="{{ url_for('make_user') }}" method="post">
-		    Username <input type="text" name="directory">
-		    <input type="submit" value="Create">
-		</form>
-		<li>
-			Create a subdirectory under the selected Folder: 
-		</li>		
-		<br>
-		<form action="{{ url_for('make_subdir') }}" method="post">
-			<select name="usernames" id="usernames">
-				{% for user in users %}
-					<option value="{{ user }}">{{ user }}</option>
-				{% endfor %}
-			</select>
-			<br>
-			Subdirectory Name <input type="text" name="directory">
-		    <input type="submit" value="Create">
-		</form>
-		<li>Choose a folder to copy your data to, then upload your data file (must be in .h5ad format).</li>
-		<br>
-		<form action="{{ url_for('upload_file') }}" method="post" enctype="multipart/form-data">
-		Type in the name of the directory and subdirectory you wish to upload to, i.e. "USER/cells". <input type="text" name="path">
-		<br>
-		File: <input type="file" name="file"><br>
-		<input style="position:relative; top:10px;" type="submit" value="Upload">
-		</form>
-		<br>
-		<li>Take a look at your data using the file crawler link above</li>
-	</ol>
-	{% endif %}
-
 	<br>
 
 	<h1 style="padding-left:35px">

setup.py

@@ -1,8 +1,9 @@
-import os
 import codecs
-from setuptools import find_packages, setup
+import os
 import sys
 
+from setuptools import find_packages, setup
+
 if sys.version_info < (3, 6):
     sys.exit("Sorry, Python < 3.6 is not supported")
 
@@ -25,8 +26,8 @@ def get_version(rel_path):
 def parse_requirements():
     reqs = []
     with open("requirements.txt", "r") as f:
-        for l in f.readlines():
+        for line in f.readlines():
-            reqs.append(l.strip("\n"))
+            reqs.append(line.strip("\n"))
     return reqs
 
 
@@ -49,7 +50,7 @@ setup(
     license="MIT",
     keywords="visualization, genomics",
     url="http://github.com/Novartis/cellxgene-gateway",
-    packages=["cellxgene_gateway"],
+    packages=find_packages(),
     package_data={
         "cellxgene_gateway": [
             "static/css/homepagestyle.css",

tests/test_cache_entry.py

@@ -1,8 +1,14 @@
 import unittest
 from cellxgene_gateway.cache_entry import CacheEntry, CacheEntryStatus
 from cellxgene_gateway.cache_key import CacheKey
+from cellxgene_gateway.items.item import ItemType
+from cellxgene_gateway.items.file.fileitem import FileItem
+from cellxgene_gateway.items.file.fileitem_source import FileItemSource
 
-key = CacheKey("czi/pbmc3k.h5ad", "pbmc3k.h5ad", "tmp.csv")
+key = CacheKey(
+    FileItem("/czi/", "pbmc3k.h5ad", ItemType.h5ad),
+    FileItemSource("/tmp", "local"),
+)
 
 
 class TestRenderEntry(unittest.TestCase):
@@ -14,16 +20,14 @@ class TestRenderEntry(unittest.TestCase):
         actual = CacheEntry.for_key(key, 8000).rewrite_text_content(
             "src:url(/static/assets/"
         )
-        expected = (
+        expected = "src:url(http://localhost:5005/source/local/view/czi/pbmc3k.h5ad/static/assets/"
-            "src:url(http://localhost:5005/view/czi/pbmc3k.h5ad/static/assets/"
-        )
         self.assertEqual(actual, expected)
 
     def test_GIVEN_absolute_src_THEN_include_path(self):
         actual = CacheEntry.for_key(key, 8000).rewrite_text_content(
             '<link rel="shortcut icon" href="/static/assets/favicon.ico">'
         )
-        expected = '<link rel="shortcut icon" href="http://localhost:5005/view/czi/pbmc3k.h5ad/static/assets/favicon.ico">'
+        expected = '<link rel="shortcut icon" href="http://localhost:5005/source/local/view/czi/pbmc3k.h5ad/static/assets/favicon.ico">'
         self.assertEqual(actual, expected)
 
 

tests/test_dir_util.py

@@ -1,49 +0,0 @@
-import unittest
-from unittest.mock import MagicMock, patch
-from cellxgene_gateway.filecrawl import render_entry
-
-
-class TestRenderEntry(unittest.TestCase):
-    def test_GIVEN_path_both_slash_THEN_view_has_single_slash(self):
-        entry = {
-            "path": "/somepath/",
-            "name": "entry",
-            "type": "file",
-            "annotations": [],
-            "children": [],
-        }
-        rendered = render_entry(entry)
-        self.assertIn("view/somepath", rendered)
-
-    def test_GIVEN_path_starts_slash_THEN_view_has_single_slash(self):
-        entry = {
-            "path": "/somepath",
-            "name": "entry",
-            "type": "file",
-            "annotations": [],
-            "children": [],
-        }
-        rendered = render_entry(entry)
-        self.assertIn("view/somepath", rendered)
-
-    def test_GIVEN_path_ends_slash_THEN_view_has_single_slash(self):
-        entry = {
-            "path": "somepath/",
-            "name": "entry",
-            "type": "file",
-            "annotations": [],
-            "children": [],
-        }
-        rendered = render_entry(entry)
-        self.assertIn("view/somepath", rendered)
-
-    def test_GIVEN_path_no_slash_THEN_view_has_single_slash(self):
-        entry = {
-            "path": "somepath",
-            "name": "entry",
-            "type": "file",
-            "annotations": [],
-            "children": [],
-        }
-        rendered = render_entry(entry)
-        self.assertIn("view/somepath", rendered)

tests/test_filecrawl.py

@@ -1,46 +1,33 @@
 import unittest
 from unittest.mock import MagicMock, patch
 
-from cellxgene_gateway.filecrawl import render_entry
+from cellxgene_gateway.filecrawl import render_item
+from cellxgene_gateway.items.item import ItemType
+from cellxgene_gateway.items.file.fileitem import FileItem
+from cellxgene_gateway.items.file.fileitem_source import FileItemSource
+
+source = FileItemSource("/tmp")
 
 
 class TestRenderEntry(unittest.TestCase):
     def test_GIVEN_path_both_slash_THEN_view_has_single_slash(self):
-        entry = {
+        entry = FileItem(
-            "path": "/somepath/",
+            subpath="/somepath/", name="entry", type=ItemType.h5ad
-            "name": "entry",
+        )
-            "type": "file",
+        rendered = render_item(entry, source)
-            "annotations": [],
+        self.assertIn("view/somepath/entry/'", rendered)
-        }
-        rendered = render_entry(entry)
-        self.assertIn("view/somepath", rendered)
 
     def test_GIVEN_path_starts_slash_THEN_view_has_single_slash(self):
-        entry = {
+        entry = FileItem(subpath="/somepath", name="entry", type=ItemType.h5ad)
-            "path": "/somepath",
+        rendered = render_item(entry, source)
-            "name": "entry",
+        self.assertIn("view/somepath/entry/'", rendered)
-            "type": "file",
-            "annotations": [],
-        }
-        rendered = render_entry(entry)
-        self.assertIn("view/somepath", rendered)
 
     def test_GIVEN_path_ends_slash_THEN_view_has_single_slash(self):
-        entry = {
+        entry = FileItem(subpath="somepath/", name="entry", type=ItemType.h5ad)
-            "path": "somepath/",
+        rendered = render_item(entry, source)
-            "name": "entry",
+        self.assertIn("view/somepath/entry/'", rendered)
-            "type": "file",
-            "annotations": [],
-        }
-        rendered = render_entry(entry)
-        self.assertIn("view/somepath", rendered)
 
     def test_GIVEN_path_no_slash_THEN_view_has_single_slash(self):
-        entry = {
+        entry = FileItem(subpath="somepath", name="entry", type=ItemType.h5ad)
-            "path": "somepath",
+        rendered = render_item(entry, source)
-            "name": "entry",
+        self.assertIn("view/somepath/entry/'", rendered)
-            "type": "file",
-            "annotations": [],
-        }
-        rendered = render_entry(entry)
-        self.assertIn("view/somepath", rendered)