From 6607b15085e05a76c17a529968619089363712c2 Mon Sep 17 00:00:00 2001 From: Andreas Eisenbarth Date: Wed, 12 Oct 2022 16:57:01 +0200 Subject: [PATCH] Exclude directories having no h5ad files --- cellxgene_gateway/items/file/fileitem_source.py | 5 ++++- cellxgene_gateway/items/s3/s3item_source.py | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cellxgene_gateway/items/file/fileitem_source.py b/cellxgene_gateway/items/file/fileitem_source.py index 22f6ce4..1028ef4 100644 --- a/cellxgene_gateway/items/file/fileitem_source.py +++ b/cellxgene_gateway/items/file/fileitem_source.py @@ -63,7 +63,7 @@ class FileItemSource(ItemSource): return item_tree - def scan_directory(self, subpath="") -> dict: + def scan_directory(self, subpath: str = "") -> ItemTree: base_path = os.path.join(self.base_path, subpath) if not os.path.exists(base_path): @@ -100,6 +100,9 @@ class FileItemSource(ItemSource): branches = [ self.scan_directory(os.path.join(subpath, subdir)) for subdir in subdirs ] + # Exclude branches without files as leaves. Since traversal is applied pre-order, + # branch.branches has already been processed and we don't need to check deeper nesting. + branches = [branch for branch in branches if branch.items or branch.branches] return ItemTree(subpath, items, branches) diff --git a/cellxgene_gateway/items/s3/s3item_source.py b/cellxgene_gateway/items/s3/s3item_source.py index a98fc16..0a18ad5 100644 --- a/cellxgene_gateway/items/s3/s3item_source.py +++ b/cellxgene_gateway/items/s3/s3item_source.py @@ -116,6 +116,7 @@ class S3ItemSource(ItemSource): branches = None if len(subdir_keys) > 0: branches = [self.scan_directory(key) for key in subdir_keys] + branches = [branch for branch in branches if branch.items or branch.branches] return ItemTree(directory_key, items, branches)