fork/cellxgene-gateway
1
0
Fork 0
mirror of https://github.com/Novartis/cellxgene-gateway.git synced 2026-10-11 04:00:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

#45 handle keys as full subpath rather than last path element

Browse Source
This commit is contained in:
Alok Saldanha
2021-04-20 05:57:48 -04:00
parent dba13ccacd
commit 1200629d74
1 changed files with 14 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cellxgene_gateway/items/s3/s3item_source.py
+14 -14
View File
@@ -38,8 +38,8 @@ class S3ItemSource(ItemSource):
self.annotation_dir_suffix = annotation_dir_suffix self.annotation_dir_suffix = annotation_dir_suffix
self.annotation_file_suffix = annotation_file_suffix self.annotation_file_suffix = annotation_file_suffix
def url(self, path): def url(self, key):
return "s3://" + join(self.bucket, path) return "s3://" + self.bucket + "/" + key
@property @property
def name(self): def name(self):
@@ -64,8 +64,8 @@ class S3ItemSource(ItemSource):
item_tree = self.scan_directory() item_tree = self.scan_directory()
return item_tree return item_tree
def scan_directory(self, subpath="") -> dict: def scan_directory(self, directory_key="") -> dict:
url = self.url(subpath) url = self.url(directory_key)
if not self.s3.exists(url): if not self.s3.exists(url):
raise Exception(f"S3 url '{url}' does not exist.") raise Exception(f"S3 url '{url}' does not exist.")
@@ -78,32 +78,32 @@ class S3ItemSource(ItemSource):
def is_annotation_dir(dir_s3key): def is_annotation_dir(dir_s3key):
return ( return (
dir_s3key.endswith(self.annotation_dir_suffix) dir_s3key.endswith(self.annotation_dir_suffix)
and self.convert_annotation_key_to_h5ad(dir_s3key) in h5ad_paths and self.convert_annotation_key_to_h5ad(dir_s3key) in h5ad_keys
) )
h5ad_paths = [ h5ad_keys = [
filepath filepath
for filepath, item_url in s3key_map.items() for filepath, item_url in s3key_map.items()
if self.is_h5ad_url(item_url) if self.is_h5ad_url(item_url)
] ]
subdirs = [ subdir_keys = [
filepath filepath
for filepath, item_url in s3key_map.items() for filepath, item_url in s3key_map.items()
if self.s3.isdir(item_url) and not is_annotation_dir(filepath) if self.s3.isdir(item_url) and not is_annotation_dir(filepath)
] ]
items = [ items = [
self.make_s3item_from_key(filename, join(subpath, filename)) self.make_s3item_from_key(
for filename in h5ad_paths key[key.rindex("/") + 1 :] if "/" in key else key, key
)
for key in h5ad_keys
] ]
branches = None branches = None
if len(subdirs) > 0: if len(subdir_keys) > 0:
branches = [ branches = [self.scan_directory(key) for key in subdir_keys]
self.scan_directory(join(subpath, subdir)) for subdir in subdirs
]
return ItemTree(subpath, items, branches) return ItemTree(directory_key, items, branches)
def create_annotation(self, item: S3Item, name: str) -> S3Item: def create_annotation(self, item: S3Item, name: str) -> S3Item:
annotation = self.make_s3item_from_key( annotation = self.make_s3item_from_key(