Add URL data locators to launch sub-command (#920)

* initial commit of URL support for launch

* lint

* modify tests to use new data locator

* add locator unit tests

* fix typo in faq

* more lint

* update faq per PR review

server/app/driver/driver.py

@@ -46,7 +46,7 @@ class CXGDriver(metaclass=ABCMeta):
         return features
 
     @abstractmethod
-    def _load_data(self, data):
+    def _load_data(self, data_locator):
         pass
 
     @abstractmethod

server/app/scanpy_engine/scanpy_engine.py

@@ -177,11 +177,16 @@ class ScanpyEngine(CXGDriver):
             }
             self.schema["layout"]["obs"].append(layout_schema)
 
-    def _load_data(self, data):
-        # as of AnnData 0.6.19, backed mode performs initial load fast, but at the
-        # cost of significantly slower access to X data.
+    def _load_data(self, data_locator):
         try:
-            self.data = anndata.read_h5ad(data)
+            # there is no guarantee data_locator indicates a local file.  The AnnData
+            # API will only consume local file objects.  If we get a non-local object,
+            # make a copy in tmp, and delete it after we load into memory.
+            with data_locator.local_handle() as lh:
+                # as of AnnData 0.6.19, backed mode performs initial load fast, but at the
+                # cost of significantly slower access to X data.
+                self.data = anndata.read_h5ad(lh)
+
         except ValueError:
             raise ScanpyFileError(
                 "File must be in the .h5ad format. Please read "
@@ -191,12 +196,11 @@ class ScanpyEngine(CXGDriver):
                 "information."
             )
         except MemoryError:
-            raise ScanpyFileError("Error while loading file: out of memory, file is too large"
-                                  " for memory available")
+            raise ScanpyFileError("Out of memory - file is too large for available memory.")
         except Exception as e:
             raise ScanpyFileError(
-                f"Error while loading file: {e}, File must be in the .h5ad format, please check "
-                f"that your input and try again."
+                f"{e} - file not found or is inaccessible.  File must be an .h5ad object.  "
+                f"Please check your input and try again."
             )
 
     @requires_data

server/app/util/data_locator.py

@@ -0,0 +1,84 @@
+import os
+import tempfile
+import fsspec
+
+
+class DataLocator():
+    """
+    DataLocator is a simple wrapper around fsspec functionality, and provides a
+    set of functions to encapsulate a data location (URI or path), interogate
+    metadata about the object at that location (size, existance, etc) and
+    access the underlying data.
+
+    https://filesystem-spec.readthedocs.io/en/latest/index.html
+
+    Example:
+        dl = DataLocator("/tmp/foo.h5ad")
+        if dl.exists():
+            print(dl.size())
+            with dl.open() as f:
+                thecontents = f.read()
+
+    DataLocator will accept a URI or native path.  Error handling is as defined
+    in fsspec.
+
+    """
+
+    def __init__(self, uri_or_path):
+        self.uri_or_path = uri_or_path
+        self.protocol, self.path = DataLocator._get_protocol_and_path(uri_or_path)
+        # work-around for LocalFileSystem not treating file: and None as the same scheme/protocol
+        self.cname = self.path if self.protocol == 'file' else self.uri_or_path
+        # will throw RuntimeError if the protocol is unsupported
+        self.fs = fsspec.filesystem(self.protocol)
+
+    @staticmethod
+    def _get_protocol_and_path(uri_or_path):
+        if "://" in uri_or_path:
+            protocol, path = uri_or_path.split("://", 1)
+            # windows!!!  Ignore single letter drive identifiers,
+            # eg, G:\foo.txt
+            if len(protocol) > 1:
+                return protocol, path
+        return None, uri_or_path
+
+    def exists(self):
+        return self.fs.exists(self.cname)
+
+    def size(self):
+        return self.fs.size(self.cname)
+
+    def isfile(self):
+        return self.fs.isfile(self.cname)
+
+    def open(self, *args):
+        return self.fs.open(self.uri_or_path, *args)
+
+    def islocal(self):
+        return self.protocol is None or self.protocol == 'file'
+
+    def local_handle(self):
+        if self.islocal():
+            return LocalFilePath(self.path)
+
+        # if not local, create a tmp file system object to contain the data,
+        # and clean it up when done.
+        with self.open() as src, tempfile.NamedTemporaryFile(prefix="cellxgene_", delete=False) as tmp:
+            tmp.write(src.read())
+            tmp.close()
+            src.close()
+            tmp_path = tmp.name
+            return LocalFilePath(tmp_path, delete=True)
+
+
+class LocalFilePath():
+    def __init__(self, tmp_path, delete=False):
+        self.tmp_path = tmp_path
+        self.delete = delete
+
+    def __enter__(self):
+        return self.tmp_path
+
+    def __exit__(self, *args):
+        if self.delete:
+            os.unlink(self.tmp_path)