Use DataLocator consistently on backend (#1208)

* allow DataLocator to accept another locator as init param

* migrate to DataLocator

* migrate to DataLocator

* lint

* migrate to DataLocator

* add check for erroroneous use of remote path and annotations

* lint

* revert default data location - now back go CWD

* remove unused import

server/common/data_locator.py

@@ -26,10 +26,18 @@ class DataLocator:
     """
 
     def __init__(self, uri_or_path):
-        self.uri_or_path = uri_or_path
+        if isinstance(uri_or_path, DataLocator):
-        self.protocol, self.path = DataLocator._get_protocol_and_path(uri_or_path)
+            locator = uri_or_path
-        # work-around for LocalFileSystem not treating file: and None as the same scheme/protocol
+            self.uri_or_path = locator.uri_or_path
-        self.cname = self.path if self.protocol == "file" else self.uri_or_path
+            self.protocol = locator.protocol
+            self.path = locator.path
+            self.cname = locator.cname
+        else:
+            self.uri_or_path = uri_or_path
+            self.protocol, self.path = DataLocator._get_protocol_and_path(uri_or_path)
+            # work-around for LocalFileSystem not treating file: and None as the same scheme/protocol
+            self.cname = self.path if self.protocol == "file" else self.uri_or_path
+
         # will throw RuntimeError if the protocol is unsupported
         self.fs = fsspec.filesystem(self.protocol)
 

server/data_anndata/anndata_adaptor.py

@@ -14,7 +14,6 @@ from server.data_common.fbs.matrix import encode_matrix_fbs
 from server.common.utils import series_to_schema
 from server.common.constants import Axis, MAX_LAYOUTS
 from server.common.errors import PrepareError, DatasetAccessError, FilterError
-from server.common.data_locator import DataLocator
 from server.compute.scanpy import scanpy_umap
 
 anndata_version = version.parse(str(anndata.__version__)).release
@@ -38,31 +37,31 @@ class AnndataAdaptor(DataAdaptor):
         pass
 
     @staticmethod
-    def pre_load_validation(location):
+    def pre_load_validation(data_locator):
-        data_locator = DataLocator(location)
         if data_locator.islocal():
             # if data locator is local, apply file system conventions and other "cheap"
             # validation checks.  If a URI, defer until we actually fetch the data and
             # try to read it.  Many of these tests don't make sense for URIs (eg, extension-
             # based typing).
             if not data_locator.exists():
-                raise DatasetAccessError(f"{location} does not exist")
+                raise DatasetAccessError(f"{data_locator.uri_or_path} does not exist")
             if not data_locator.isfile():
-                raise DatasetAccessError(f"{location} is not a file")
+                raise DatasetAccessError(f"{data_locator.uri_or_path} is not a file")
 
     @staticmethod
-    def file_size(location):
+    def file_size(data_locator):
-        data_locator = DataLocator(location)
         return data_locator.size() if data_locator.islocal() else 0
 
     @staticmethod
-    def open(location, config):
+    def open(data_locator, config):
-        data_locator = DataLocator(location)
         return AnndataAdaptor(data_locator, config)
 
     def get_location(self):
         return self.data_locator.uri_or_path
 
+    def get_data_locator(self):
+        return self.data_locator
+
     def get_name(self):
         return "cellxgene anndata adaptor version"
 

server/data_common/data_adaptor.py

@@ -10,7 +10,6 @@ from server.common.errors import FilterError, JSONEncodingValueError
 from server.compute.diffexp import diffexp_ttest
 from server.common.utils import jsonify_numpy
 from server.common.app_config import AppFeature, AppConfig
-from server.common.data_locator import DataLocator
 
 
 class DataAdaptor(metaclass=ABCMeta):
@@ -32,17 +31,17 @@ class DataAdaptor(metaclass=ABCMeta):
 
     @staticmethod
     @abstractmethod
-    def pre_load_validation(location):
+    def pre_load_validation(data_locator):
         pass
 
     @staticmethod
     @abstractmethod
-    def open(location, config):
+    def open(data_locator, config):
         pass
 
     @staticmethod
     @abstractmethod
-    def file_size(location):
+    def file_size(data_locator):
         pass
 
     @abstractmethod
@@ -105,6 +104,10 @@ class DataAdaptor(metaclass=ABCMeta):
     def get_location(self):
         pass
 
+    @abstractmethod
+    def get_data_locator(self):
+        pass
+
     def get_about(self):
         return None
 
@@ -345,8 +348,7 @@ class DataAdaptor(metaclass=ABCMeta):
 
     def get_last_mod_time(self):
         try:
-            data_locator = DataLocator(self.get_location())
+            lastmod = self.get_data_locator().lastmodtime()
-            lastmod = data_locator.lastmodtime()
         except RuntimeError:
             lastmod = None
         return lastmod

server/data_common/matrix_loader.py

@@ -3,6 +3,7 @@ import threading
 import time
 from server.data_common.rwlock import RWLock
 from server.common.errors import DatasetAccessError
+from server.common.data_locator import DataLocator
 from contextlib import contextmanager
 
 
@@ -136,7 +137,8 @@ class MatrixDataType(Enum):
 
 class MatrixDataLoader(object):
     def __init__(self, location, etype=None):
-        self.location = location
+        """ location can be a string or DataLocator """
+        self.location = DataLocator(location)
         if etype is None:
             self.etype = self.matrix_data_type()
         else:
@@ -152,9 +154,9 @@ class MatrixDataLoader(object):
             self.matrix_type = CxgAdaptor
 
     def matrix_data_type(self):
-        if self.location.endswith(".h5ad"):
+        if self.location.path.endswith(".h5ad"):
             return MatrixDataType.H5AD
-        elif ".cxg" in self.location:
+        elif ".cxg" in self.location.path:
             return MatrixDataType.CXG
         else:
             return MatrixDataType.UNKNOWN

server/data_cxg/cxg_adaptor.py

@@ -18,13 +18,13 @@ class CxgAdaptor(DataAdaptor):
     # TODO:  The tiledb context parameters should be a configuration option
     tiledb_ctx = tiledb.Ctx({"sm.tile_cache_size": 8 * 1024 * 1024 * 1024, "sm.num_reader_threads": 32})
 
-    def __init__(self, location, config=None):
+    def __init__(self, data_locator, config=None):
         super().__init__(config)
-        self.url = location
         self.arrays = {}
         self.lock = threading.Lock()
 
-        self.url = location
+        self.data_locator = data_locator
+        self.url = data_locator.uri_or_path
         if self.url[-1] != "/":
             self.url += "/"
 
@@ -37,17 +37,18 @@ class CxgAdaptor(DataAdaptor):
         self.arrays.clear()
 
     @staticmethod
-    def pre_load_validation(location):
+    def pre_load_validation(data_locator):
+        location = data_locator.uri_or_path
         if not CxgAdaptor.isvalid(location):
             raise DatasetAccessError(f"cxg matrix is not valid: {location}")
 
     @staticmethod
-    def file_size(location):
+    def file_size(data_locator):
         return 0
 
     @staticmethod
-    def open(location, args):
+    def open(data_locator, args):
-        return CxgAdaptor(location, args)
+        return CxgAdaptor(data_locator, args)
 
     def get_about(self):
         return self.about if self.about else super().get_about()
@@ -58,6 +59,9 @@ class CxgAdaptor(DataAdaptor):
     def get_location(self):
         return self.url
 
+    def get_data_locator(self):
+        return self.data_locator
+
     def get_name(self):
         return "cellxgene cxg adaptor version"
 

server/eb/app.py

@@ -45,7 +45,8 @@ try:
         obs_names=None,
         var_names=None,
         anndata_backed=False,
-        disable_diffexp=False)
+        disable_diffexp=False,
+    )
 
     matrix_data_cache_manager = MatrixDataCacheManager()
     annotations = None