From d2b20129f7a3acf89a34128e1594aa7c02adaaa9 Mon Sep 17 00:00:00 2001 From: Emanuele Bezzi Date: Fri, 29 Jul 2022 07:05:36 -0400 Subject: [PATCH] use fsspec.download for S3 artifacts + additional logging (#2536) --- server/common/utils/data_locator.py | 5 ++--- server/data_anndata/anndata_adaptor.py | 8 ++++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/server/common/utils/data_locator.py b/server/common/utils/data_locator.py index 1c901c84..4349ed7e 100644 --- a/server/common/utils/data_locator.py +++ b/server/common/utils/data_locator.py @@ -113,10 +113,9 @@ class DataLocator: # do our best to create a file with the same. ext = os.path.splitext(self.path) suffix = None if ext[1] == "" else ext[1] - with self.open() as src, tempfile.NamedTemporaryFile(prefix="cellxgene_", suffix=suffix, delete=False) as tmp: - tmp.write(src.read()) + with tempfile.NamedTemporaryFile(prefix="cellxgene_", suffix=suffix, delete=False) as tmp: + self.fs.download(self.uri_or_path, tmp.name) tmp.close() - src.close() tmp_path = tmp.name return LocalFilePath(tmp_path, delete=True) diff --git a/server/data_anndata/anndata_adaptor.py b/server/data_anndata/anndata_adaptor.py index d71331c3..f5fbf25b 100644 --- a/server/data_anndata/anndata_adaptor.py +++ b/server/data_anndata/anndata_adaptor.py @@ -174,10 +174,14 @@ class AnndataAdaptor(DataAdaptor): except MemoryError: raise DatasetAccessError("Out of memory - file is too large for available memory.") except Exception: - raise DatasetAccessError( + import traceback + message = ( "File not found or is inaccessible. File must be an .h5ad object. " "Please check your input and try again." - ) + ) + if self.server_config.app__verbose: + message += f"\n{traceback.format_exc()}" + raise DatasetAccessError(message) def _validate_and_initialize(self): if anndata_version_is_pre_070():