Add URL data locators to launch sub-command (#920)

* initial commit of URL support for launch

* lint

* modify tests to use new data locator

* add locator unit tests

* fix typo in faq

* more lint

* update faq per PR review

server/test/test_nan_scanpy_engine.py

@@ -7,6 +7,7 @@ import decode_fbs
 
 from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
 from server.app.util.errors import FilterError
+from server.app.util.data_locator import DataLocator
 
 
 class NaNTest(unittest.TestCase):
@@ -20,12 +21,12 @@ class NaNTest(unittest.TestCase):
         }
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", category=UserWarning)
-            self.data = ScanpyEngine("server/test/test_datasets/nan.h5ad", self.args)
+            self.data = ScanpyEngine(DataLocator("server/test/test_datasets/nan.h5ad"), self.args)
             self.data._create_schema()
 
     def test_load(self):
         with self.assertWarns(UserWarning):
-            ScanpyEngine("server/test/test_datasets/nan.h5ad", self.args)
+            ScanpyEngine(DataLocator("server/test/test_datasets/nan.h5ad"), self.args)
 
     def test_init(self):
         self.assertEqual(self.data.cell_count, 100)

server/test/test_scanpy_engine.py

@@ -10,6 +10,7 @@ from pandas import Series
 
 from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
 from server.app.util.errors import FilterError
+from server.app.util.data_locator import DataLocator
 
 
 class EngineTest(unittest.TestCase):
@@ -22,7 +23,7 @@ class EngineTest(unittest.TestCase):
             "var_names": None,
             "diffexp_lfc_cutoff": 0.01,
         }
-        self.data = ScanpyEngine("example-dataset/pbmc3k.h5ad", args)
+        self.data = ScanpyEngine(DataLocator("example-dataset/pbmc3k.h5ad"), args)
 
     def test_init(self):
         self.assertEqual(self.data.cell_count, 2638)

server/test/test_scanpy_engine_csc.py

@@ -10,6 +10,7 @@ from pandas import Series
 
 from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
 from server.app.util.errors import FilterError
+from server.app.util.data_locator import DataLocator
 
 
 class EngineTest(unittest.TestCase):
@@ -21,7 +22,7 @@ class EngineTest(unittest.TestCase):
             "var_names": None,
             "diffexp_lfc_cutoff": 0.01,
         }
-        self.data = ScanpyEngine("server/test/test_datasets/pbmc3k-CSC-gz.h5ad", args)
+        self.data = ScanpyEngine(DataLocator("server/test/test_datasets/pbmc3k-CSC-gz.h5ad"), args)
 
     def test_init(self):
         self.assertEqual(self.data.cell_count, 2638)

server/test/test_scanpy_engine_csr.py

@@ -10,6 +10,7 @@ from pandas import Series
 
 from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
 from server.app.util.errors import FilterError
+from server.app.util.data_locator import DataLocator
 
 
 class EngineTest(unittest.TestCase):
@@ -21,7 +22,7 @@ class EngineTest(unittest.TestCase):
             "var_names": None,
             "diffexp_lfc_cutoff": 0.01,
         }
-        self.data = ScanpyEngine("server/test/test_datasets/pbmc3k-CSR-gz.h5ad", args)
+        self.data = ScanpyEngine(DataLocator("server/test/test_datasets/pbmc3k-CSR-gz.h5ad"), args)
 
     def test_init(self):
         self.assertEqual(self.data.cell_count, 2638)

server/test/test_scanpy_engine_data_load.py

@@ -3,11 +3,15 @@ import json
 
 from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
 from server.app.util.errors import DriverError
+from server.app.util.data_locator import DataLocator
 
 
 class DataLoadEngineTest(unittest.TestCase):
+    """
+    Test file loading, including deferred loading/update.
+    """
     def setUp(self):
-        self.data_file = "example-dataset/pbmc3k.h5ad"
+        self.data_file = DataLocator("example-dataset/pbmc3k.h5ad")
         self.data = ScanpyEngine()
 
     def test_init(self):
@@ -45,5 +49,39 @@ class DataLoadEngineTest(unittest.TestCase):
         result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
         self.assertEqual(len(result), 20)
 
-    if __name__ == "__main__":
-        unittest.main()
+
+class DataLocatorEngineTest(unittest.TestCase):
+    """
+    Test various types of data locators we expect to consume
+    """
+    def setUp(self):
+        self.args = {
+            "layout": ["umap"],
+            "max_category_items": 100,
+            "obs_names": None,
+            "var_names": None,
+            "diffexp_lfc_cutoff": 0.01,
+        }
+
+    def stdAsserts(self, data):
+        """ run these each time we load the data """
+        self.assertIsNotNone(data)
+        self.assertEqual(data.cell_count, 2638)
+        self.assertEqual(data.gene_count, 1838)
+
+    def test_posix_file(self):
+        locator = DataLocator("example-dataset/pbmc3k.h5ad")
+        data = ScanpyEngine(locator, self.args)
+        self.stdAsserts(data)
+
+    def test_url_https(self):
+        url = "https://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/example-dataset/pbmc3k.h5ad"
+        locator = DataLocator(url)
+        data = ScanpyEngine(locator, self.args)
+        self.stdAsserts(data)
+
+    def test_url_http(self):
+        url = "http://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/example-dataset/pbmc3k.h5ad"
+        locator = DataLocator(url)
+        data = ScanpyEngine(locator, self.args)
+        self.stdAsserts(data)