rebased "Introduction of ItemSource interface" patch

cellxgene_gateway/filecrawl.py

@@ -8,6 +8,7 @@
 # the specific language governing permissions and limitations under the License.
 
 import os
+import urllib.parse
 
 from flask import url_for
 
@@ -15,104 +16,55 @@ from cellxgene_gateway import env
 from cellxgene_gateway.dir_util import annotations_suffix, make_annotations, make_h5ad
 
 
-def recurse_dir(path):
-    if not os.path.exists(path):
-        raise CellxgeneException(
-            "The given path does not exist.", status.HTTP_400_BAD_REQUEST
-        )
-
-    all_entries = sorted(os.listdir(path))
-
-    def is_h5ad(el):
-        return el.endswith(".h5ad") and os.path.isfile(os.path.join(path, el))
-
-    h5ad_entries = [x for x in all_entries if is_h5ad(x)]
-    annotation_dir_entries = [
-        x
-        for x in all_entries
-        if x.endswith(annotations_suffix) and make_h5ad(x) in h5ad_entries
-    ]
-
-    def list_annotations(el):
-        full_path = os.path.join(path, el)
-        if not os.path.isdir(full_path):
-            entries = []
-        else:
-            entries = [
-                {
-                    "name": x[:-13]
-                    if (len(x) > 13 and x[-13] in ["-", "_"])
-                    else (x[:-4] if x.endswith(".csv") else x),
-                    "path": os.path.join(full_path, x + "/").replace(
-                        env.cellxgene_data, ""
-                    ),
-                }
-                for x in sorted(os.listdir(full_path))
-                if x.endswith(".csv") and os.path.isfile(os.path.join(full_path, x))
-            ]
-        return [
-            {
-                "name": "new",
-                "class": "new",
-                "path": full_path.replace(env.cellxgene_data, "").rstrip("/"),
-            }
-        ] + entries
-
-    def make_entry(el):
-        full_path = os.path.join(path, el)
-        if el in h5ad_entries:
-            return {
-                "path": full_path.replace(env.cellxgene_data, ""),
-                "name": el,
-                "type": "file",
-                "annotations": list_annotations(make_annotations(el)),
-            }
-        elif os.path.isdir(full_path) and el not in annotation_dir_entries:
-            return {
-                "path": full_path.replace(env.cellxgene_data, ""),
-                "name": el,
-                "type": "directory",
-                "children": recurse_dir(full_path),
-            }
-        else:
-            return {
-                "path": full_path,
-                "name": el,
-                "type": "neither",
-            }
-
-    return [make_entry(x) for x in all_entries]
-
-
-def render_entries(entries):
-    return "<ul>" + "\n".join([render_entry(e) for e in entries]) + "</ul>"
-
-
-def get_url(entry):
-    return url_for("do_view", path=entry["path"].lstrip("/"))
-
-
-def get_class(entry):
-    return f" class='{entry['class']}'" if "class" in entry else ""
-
-
-def render_annotations(entry):
-    if len(entry["annotations"]) > 0:
-        return " | annotations: " + ", ".join(
+def render_annotations(item, item_source):
+    url = url_for(
+        "do_view",
+        path=item_source.get_annotations_subpath(item),
+        source_name=item_source.name,
+    )
+    new_annotation = f"<a class='new' href='{url}'>new</a>"
+    annotations = (
+        ", ".join(
             [
-                f"<a href='{get_url(a)}'{get_class(a)}>{a['name']}</a>"
-                for a in entry["annotations"]
+                f"<a href='{url_for('do_view', path=a.descriptor, source_name = item_source.name)}/'>{a.name}</a>"
+                for a in item.annotations
             ]
         )
-    else:
-        return ""
+        + ", "
+        if item.annotations
+        else ""
+    )
+    return " | annotations: " + annotations + new_annotation
 
 
-def render_entry(entry):
-    if entry["type"] == "file":
-        return f"<li> <a href='{ get_url(entry) }/'>{entry['name']}</a> {render_annotations(entry)}</li>"
-    elif entry["type"] == "directory":
-        url = f"/filecrawl/{entry['path'].lstrip('/')}"
-        return f"<li><a href='{url}'>{entry['name']}</a>{render_entries(entry['children'])}</li>"
+def render_item(item, item_source):
+    url = url_for("do_view", path=item.descriptor, source_name=item_source.name) + "/"
+    item_string = f"<li> <a href='{ url }'>{item.name}</a> {render_annotations(item, item_source)}</li>"
+    return item_string
+
+
+def render_item_tree(item_tree, item_source):
+    items = (
+        "\n".join([render_item(i, item_source) for i in item_tree.items])
+        if item_tree.items
+        else ""
+    )
+    branches = (
+        "\n".join([render_item_tree(b, item_source) for b in item_tree.branches])
+        if item_tree.branches
+        else ""
+    )
+    html = "<ul>" + items + branches + "</ul>"
+    if item_tree.descriptor:
+        descriptor = item_tree.descriptor.lstrip("/")
+        url = f"/filecrawl/{descriptor}?source={item_source.name}"
+        name = descriptor.rsplit("/")[1] if descriptor.find("/") >= 0 else descriptor
+        return f"<li><a href='{url}'>{name}</a>{html}</li>"
     else:
-        return ""
+        return html
+
+
+def render_item_source(item_source, filter=None):
+    item_tree = item_source.list_items(filter)
+    heading = f"<h6><a href='/filecrawl.html?source={urllib.parse.quote_plus(item_source.name)}'>{item_source.name}</a></h6>"
+    return heading + render_item_tree(item_tree, item_source)