From 03a75ec56666c0c2d77cadbde3538c2f6a552d79 Mon Sep 17 00:00:00 2001 From: bmccandless Date: Fri, 17 Apr 2020 10:14:24 -0700 Subject: [PATCH] s3 listings were cached, preventing new datasets from being accessible (#1408) Fixes #1407 --- server/common/data_locator.py | 8 ++++++-- server/requirements.txt | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/server/common/data_locator.py b/server/common/data_locator.py index 50117682..bf47d724 100644 --- a/server/common/data_locator.py +++ b/server/common/data_locator.py @@ -42,8 +42,12 @@ class DataLocator: self.cname = self.path if self.protocol == "file" else self.uri_or_path # fsspec.filesystem will throw RuntimeError if the protocol is unsupported - if self.protocol == "s3" and region_name: - self.fs = fsspec.filesystem(self.protocol, config_kwargs={"region_name": region_name}) + if self.protocol == "s3": + if region_name: + config_kwargs = dict(region_name=region_name) + self.fs = fsspec.filesystem(self.protocol, listings_expiry_time=30, config_kwargs=config_kwargs) + else: + self.fs = fsspec.filesystem(self.protocol, listings_expiry_time=30) else: self.fs = fsspec.filesystem(self.protocol) diff --git a/server/requirements.txt b/server/requirements.txt index 43982c2b..603929f6 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -18,4 +18,4 @@ PyYAML>=5.3 scipy>=1.3.0 requests>=2.22.0 tiledb>=0.5.3 -s3fs>=0.4.0,!=0.4.1 +s3fs>=0.4.2