fork/cellxgene-gateway
1
0
Fork 0
mirror of https://github.com/Novartis/cellxgene-gateway.git synced 2026-09-15 12:47:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

add environment variable S3_DISABLE_LISTINGS_CACHE per Alok's request

Browse Source
This commit is contained in:
Alex Rogozhnikov
2022-03-14 09:59:55 -07:00
parent 893b2f1af1
commit 3e3bd22512
1 changed files with 7 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
cellxgene_gateway/items/s3/s3item_source.py
View File

@@ -28,7 +28,11 @@ class S3ItemSource(ItemSource):
annotation_file_suffix=".csv",
):
self._name = name
self.s3 = s3fs.S3FileSystem(use_listings_cache=False)
disable_cache = os.environ.get("S3_DISABLE_LISTINGS_CACHE", "false").lower()
assert disable_cache in ['0', '1', 'false', 'true']
self.use_listings_cache = disable_cache.lower() not in ["0", "false"]
self.s3 = s3fs.S3FileSystem(use_listings_cache=self.use_listings_cache)
if bucket.startswith("s3://"):
raise Exception(
f"Bucket name should not include s3:// prefix, got {bucket}"
@@ -75,7 +79,7 @@ class S3ItemSource(ItemSource):
s3key_map = dict(
(self.remove_bucket(filepath), "s3://" + filepath)
for filepath in sorted(self.s3.ls(url, refresh=True))
for filepath in sorted(self.s3.ls(url, refresh=not self.use_listings_cache))
)
def is_annotation_dir(dir_s3key):
@@ -166,7 +170,7 @@ class S3ItemSource(ItemSource):
self.make_s3item_from_key(
basename(annotation), self.remove_bucket(annotation), True
)
for annotation in sorted(self.s3.ls(annotations_fullpath))
for annotation in sorted(self.s3.ls(annotations_fullpath, refresh=not self.use_listings_cache))
if annotation.endswith(self.annotation_file_suffix)
and self.s3.isfile("s3://" + annotation)
]