experimental re-embedding (#1186)

* first cut at re-embedding route and back-end support

* update and expand config route tests

* add scanpy_umap

* add reembedding to config route parameters

* front-end support for reembedding fetch and UI

* remove unused imports

* add loading state

* save reembedding in reducer state

* improve withColsFrom

* transmit reembed schema to client; pick unique embedding names

* display embeddings

* format

* lint

* spaces, tab size 2

* lint

* test hack for smoke-test race

* back out hack sleep

* add check for backed mode

* add unit test for reembedding

* lint

* hide re-embedding CLI param from help

server/test/test_anndata_adaptor.py

@@ -3,6 +3,7 @@ from os import path
 import pytest
 import time
 import unittest
+import sys
 import server.test.decode_fbs as decode_fbs
 from parameterized import parameterized_class
 
@@ -97,7 +98,21 @@ class AdaptorTest(unittest.TestCase):
             self.data._create_schema()
 
     def test_config(self):
-        self.assertEqual(self.data.get_features()["layout_obs"].available, True)
+        features = self.data.get_features(annotations=None)
+
+        # test each for singular presence and accuracy of available flag
+        def check_feature(method, path, available):
+            feature = list(
+                filter(lambda f: f.method == method and f.path == path and f.available == available, features)
+            )
+            self.assertIsNotNone(feature)
+            self.assertEqual(len(feature), 1)
+
+        check_feature("POST", "/cluster/", False)
+        check_feature("POST", "/diffexp/", not self.data.config.disable_diffexp)
+        check_feature("GET", "/layout/obs", True)
+        check_feature("PUT", "/layout/obs", self.data.config.enable_reembedding)
+        check_feature("PUT", "/annotations/obs", False)
 
     def test_layout(self):
         fbs = self.data.layout_to_fbs_matrix()
@@ -185,3 +200,39 @@ class AdaptorTest(unittest.TestCase):
         self.assertEqual(data["n_rows"], 2638)
         self.assertEqual(data["n_cols"], 3)
         self.assertTrue((data["col_idx"] == [15, 1818, 1837]).all())
+
+    def test_compute_embedding(self):
+        filter = {"obs": {"index": [[0, 100]]}}
+
+        # Verify that we correctly handle the case where we lack scanpy
+        import unittest.mock
+
+        with unittest.mock.patch.dict(sys.modules, {"scanpy": None}):
+            with self.assertRaises(NotImplementedError):
+                self.data.compute_embedding("umap", filter)
+
+        # if we happen to have scanpy, test the full API, else punt
+        import importlib
+
+        scanpy_spec = importlib.util.find_spec("scanpy")
+        if scanpy_spec is None:
+            print("Skipping compute_embedding test as ScanPy not installed")
+            return
+
+        # this feature is unsupported in backed mode, and we expect an error
+        if self.data.data.isbacked:
+            with self.assertRaises(NotImplementedError):
+                self.data.compute_embedding("umap", filter)
+            return
+
+        (schema, fbs) = self.data.compute_embedding("umap", filter)
+
+        self.assertIsInstance(schema["name"], str)
+        name = schema["name"]
+        self.assertEqual(schema["type"], "float32")
+        self.assertEqual(schema["dims"], [f"{name}_0", f"{name}_1"])
+
+        emb = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(emb["n_rows"], 100)
+        self.assertEqual(emb["n_cols"], 2)
+        self.assertEqual(emb["col_idx"], [f"{name}_0", f"{name}_1"])

server/test/test_api.py

@@ -44,7 +44,7 @@ class EndPoints(object):
         result_data = result.json()
         self.assertIn("library_versions", result_data["config"])
         self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k")
-        self.assertEqual(len(result_data["config"]["features"]), 3)
+        self.assertEqual(len(result_data["config"]["features"]), 5)
 
     def test_get_layout_fbs(self):
         endpoint = "layout/obs"

server/test/test_writable_annotation.py

@@ -136,3 +136,20 @@ class WritableAnnotationTest(unittest.TestCase):
             all_col_schema["cat_B"],
             {"name": "cat_B", "type": "categorical", "categories": ["label_B"], "writable": True},
         )
+
+    def test_config(self):
+        features = self.data.get_features(self.annotations)
+
+        # test each for singular presence and accuracy of available flag
+        def check_feature(method, path, available):
+            feature = list(
+                filter(lambda f: f.method == method and f.path == path and f.available == available, features)
+            )
+            self.assertIsNotNone(feature)
+            self.assertEqual(len(feature), 1)
+
+        check_feature("POST", "/cluster/", False)
+        check_feature("POST", "/diffexp/", not self.data.config.disable_diffexp)
+        check_feature("GET", "/layout/obs", True)
+        check_feature("PUT", "/layout/obs", self.data.config.enable_reembedding)
+        check_feature("PUT", "/annotations/obs", True)