Merge pull request #77 from aeisenbarth/filter-empty-folders

Filter directories without h5ad files

cellxgene_gateway/items/file/fileitem_source.py

@@ -68,7 +68,7 @@ class FileItemSource(ItemSource):
 
         return item_tree
 
-    def scan_directory(self, subpath="") -> dict:
+    def scan_directory(self, subpath: str = "") -> ItemTree:
         base_path = os.path.join(self.base_path, subpath)
 
         if not os.path.exists(base_path):
@@ -105,6 +105,11 @@ class FileItemSource(ItemSource):
             branches = [
                 self.scan_directory(os.path.join(subpath, subdir)) for subdir in subdirs
             ]
+            # Exclude branches without files as leaves. Since traversal is applied pre-order,
+            # branch.branches has already been processed and we don't need to check deeper nesting.
+            branches = [
+                branch for branch in branches if branch.items or branch.branches
+            ]
 
         return ItemTree(subpath, items, branches)
 

cellxgene_gateway/items/s3/s3item_source.py

@@ -116,6 +116,9 @@ class S3ItemSource(ItemSource):
         branches = None
         if len(subdir_keys) > 0:
             branches = [self.scan_directory(key) for key in subdir_keys]
+            branches = [
+                branch for branch in branches if branch.items or branch.branches
+            ]
 
         return ItemTree(directory_key, items, branches)
 

tests/test_filecrawl.py

@@ -1,4 +1,5 @@
 import unittest
+from collections import defaultdict
 from unittest.mock import MagicMock, patch
 
 from cellxgene_gateway.filecrawl import (
@@ -48,12 +49,48 @@ class TestRenderItemSource(unittest.TestCase):
 
 
 class TestRenderItemTree(unittest.TestCase):
+    def setUp(self):
+        from cellxgene_gateway.gateway import app
+
+        self.app = app
+        self.app_context = self.app.test_request_context()
+        self.app_context.push()
+
     @patch("cellxgene_gateway.items.file.fileitem_source.FileItemSource")
     def test_GIVEN_deep_nested_dirs_THEN_includes_dirs_in_output(self, item_source):
         item_source.name = "FakeSource"
-        item_tree = ItemTree("foo/bar/baz", [], [])
+        item_source.get_annotations_subpath = lambda _: "FakeAnnotations"
+        file_item = FileItem(
+            subpath="foo/bar/baz", name="file.h5ad", type=ItemType.h5ad
+        )
+        item_tree = ItemTree("foo/bar/baz", [file_item], [])
         rendered = render_item_tree(item_tree, item_source)
         self.assertEqual(
             rendered,
-            "<li><a href='/filecrawl/foo/bar/baz?source=FakeSource'>baz</a><ul></ul></li>",
+            "<li><a href='/filecrawl/foo/bar/baz?source=FakeSource'>baz</a><ul>"
+            "<li> <a href='/source/FakeSource/view/foo/bar/baz/file.h5ad/'>file.h5ad</a>"
+            "  | annotations: <a class='new' href='/source/FakeSource/view/FakeAnnotations'>new</a>"
+            "</li></ul></li>",
+        )
+
+    @patch(
+        "os.listdir",
+        side_effect=lambda parent: defaultdict(
+            list, {"tmp": ["foo"], "tmp/foo": ["bar"]}
+        )[parent],
+    )
+    @patch("os.path.exists", return_value=True)
+    def test_GIVEN_dirs_without_h5ad_THEN_excludes_dirs_in_output(
+        self, listdir, exists
+    ):
+        # Directories:
+        # - tmp
+        #   - foo
+        #     - bar (no h5ad files)
+        item_source = FileItemSource("tmp", name="local")
+        item_tree = item_source.list_items("foo")
+        rendered = render_item_tree(item_tree, item_source)
+        self.assertEqual(
+            rendered,
+            "<li><a href='/filecrawl/foo?source=local'>foo</a><ul></ul></li>",
         )