diff --git a/server/common/data_locator.py b/server/common/data_locator.py index 07f3ad55..45eaadbe 100644 --- a/server/common/data_locator.py +++ b/server/common/data_locator.py @@ -26,10 +26,18 @@ class DataLocator: """ def __init__(self, uri_or_path): - self.uri_or_path = uri_or_path - self.protocol, self.path = DataLocator._get_protocol_and_path(uri_or_path) - # work-around for LocalFileSystem not treating file: and None as the same scheme/protocol - self.cname = self.path if self.protocol == "file" else self.uri_or_path + if isinstance(uri_or_path, DataLocator): + locator = uri_or_path + self.uri_or_path = locator.uri_or_path + self.protocol = locator.protocol + self.path = locator.path + self.cname = locator.cname + else: + self.uri_or_path = uri_or_path + self.protocol, self.path = DataLocator._get_protocol_and_path(uri_or_path) + # work-around for LocalFileSystem not treating file: and None as the same scheme/protocol + self.cname = self.path if self.protocol == "file" else self.uri_or_path + # will throw RuntimeError if the protocol is unsupported self.fs = fsspec.filesystem(self.protocol) diff --git a/server/data_anndata/anndata_adaptor.py b/server/data_anndata/anndata_adaptor.py index b96b4c7f..7f39c81a 100644 --- a/server/data_anndata/anndata_adaptor.py +++ b/server/data_anndata/anndata_adaptor.py @@ -14,7 +14,6 @@ from server.data_common.fbs.matrix import encode_matrix_fbs from server.common.utils import series_to_schema from server.common.constants import Axis, MAX_LAYOUTS from server.common.errors import PrepareError, DatasetAccessError, FilterError -from server.common.data_locator import DataLocator from server.compute.scanpy import scanpy_umap anndata_version = version.parse(str(anndata.__version__)).release @@ -38,31 +37,31 @@ class AnndataAdaptor(DataAdaptor): pass @staticmethod - def pre_load_validation(location): - data_locator = DataLocator(location) + def pre_load_validation(data_locator): if data_locator.islocal(): # if data locator is local, apply file system conventions and other "cheap" # validation checks. If a URI, defer until we actually fetch the data and # try to read it. Many of these tests don't make sense for URIs (eg, extension- # based typing). if not data_locator.exists(): - raise DatasetAccessError(f"{location} does not exist") + raise DatasetAccessError(f"{data_locator.uri_or_path} does not exist") if not data_locator.isfile(): - raise DatasetAccessError(f"{location} is not a file") + raise DatasetAccessError(f"{data_locator.uri_or_path} is not a file") @staticmethod - def file_size(location): - data_locator = DataLocator(location) + def file_size(data_locator): return data_locator.size() if data_locator.islocal() else 0 @staticmethod - def open(location, config): - data_locator = DataLocator(location) + def open(data_locator, config): return AnndataAdaptor(data_locator, config) def get_location(self): return self.data_locator.uri_or_path + def get_data_locator(self): + return self.data_locator + def get_name(self): return "cellxgene anndata adaptor version" diff --git a/server/data_common/data_adaptor.py b/server/data_common/data_adaptor.py index c2366432..e3607f81 100644 --- a/server/data_common/data_adaptor.py +++ b/server/data_common/data_adaptor.py @@ -10,7 +10,6 @@ from server.common.errors import FilterError, JSONEncodingValueError from server.compute.diffexp import diffexp_ttest from server.common.utils import jsonify_numpy from server.common.app_config import AppFeature, AppConfig -from server.common.data_locator import DataLocator class DataAdaptor(metaclass=ABCMeta): @@ -32,17 +31,17 @@ class DataAdaptor(metaclass=ABCMeta): @staticmethod @abstractmethod - def pre_load_validation(location): + def pre_load_validation(data_locator): pass @staticmethod @abstractmethod - def open(location, config): + def open(data_locator, config): pass @staticmethod @abstractmethod - def file_size(location): + def file_size(data_locator): pass @abstractmethod @@ -105,6 +104,10 @@ class DataAdaptor(metaclass=ABCMeta): def get_location(self): pass + @abstractmethod + def get_data_locator(self): + pass + def get_about(self): return None @@ -345,8 +348,7 @@ class DataAdaptor(metaclass=ABCMeta): def get_last_mod_time(self): try: - data_locator = DataLocator(self.get_location()) - lastmod = data_locator.lastmodtime() + lastmod = self.get_data_locator().lastmodtime() except RuntimeError: lastmod = None return lastmod diff --git a/server/data_common/matrix_loader.py b/server/data_common/matrix_loader.py index 6b1659ab..41a32f7d 100644 --- a/server/data_common/matrix_loader.py +++ b/server/data_common/matrix_loader.py @@ -3,6 +3,7 @@ import threading import time from server.data_common.rwlock import RWLock from server.common.errors import DatasetAccessError +from server.common.data_locator import DataLocator from contextlib import contextmanager @@ -136,7 +137,8 @@ class MatrixDataType(Enum): class MatrixDataLoader(object): def __init__(self, location, etype=None): - self.location = location + """ location can be a string or DataLocator """ + self.location = DataLocator(location) if etype is None: self.etype = self.matrix_data_type() else: @@ -152,9 +154,9 @@ class MatrixDataLoader(object): self.matrix_type = CxgAdaptor def matrix_data_type(self): - if self.location.endswith(".h5ad"): + if self.location.path.endswith(".h5ad"): return MatrixDataType.H5AD - elif ".cxg" in self.location: + elif ".cxg" in self.location.path: return MatrixDataType.CXG else: return MatrixDataType.UNKNOWN diff --git a/server/data_cxg/cxg_adaptor.py b/server/data_cxg/cxg_adaptor.py index da4c71a0..71775edd 100644 --- a/server/data_cxg/cxg_adaptor.py +++ b/server/data_cxg/cxg_adaptor.py @@ -18,13 +18,13 @@ class CxgAdaptor(DataAdaptor): # TODO: The tiledb context parameters should be a configuration option tiledb_ctx = tiledb.Ctx({"sm.tile_cache_size": 8 * 1024 * 1024 * 1024, "sm.num_reader_threads": 32}) - def __init__(self, location, config=None): + def __init__(self, data_locator, config=None): super().__init__(config) - self.url = location self.arrays = {} self.lock = threading.Lock() - self.url = location + self.data_locator = data_locator + self.url = data_locator.uri_or_path if self.url[-1] != "/": self.url += "/" @@ -37,17 +37,18 @@ class CxgAdaptor(DataAdaptor): self.arrays.clear() @staticmethod - def pre_load_validation(location): + def pre_load_validation(data_locator): + location = data_locator.uri_or_path if not CxgAdaptor.isvalid(location): raise DatasetAccessError(f"cxg matrix is not valid: {location}") @staticmethod - def file_size(location): + def file_size(data_locator): return 0 @staticmethod - def open(location, args): - return CxgAdaptor(location, args) + def open(data_locator, args): + return CxgAdaptor(data_locator, args) def get_about(self): return self.about if self.about else super().get_about() @@ -58,6 +59,9 @@ class CxgAdaptor(DataAdaptor): def get_location(self): return self.url + def get_data_locator(self): + return self.data_locator + def get_name(self): return "cellxgene cxg adaptor version" diff --git a/server/eb/app.py b/server/eb/app.py index 7bca6b49..98cdfe90 100644 --- a/server/eb/app.py +++ b/server/eb/app.py @@ -45,7 +45,8 @@ try: obs_names=None, var_names=None, anndata_backed=False, - disable_diffexp=False) + disable_diffexp=False, + ) matrix_data_cache_manager = MatrixDataCacheManager() annotations = None