experimental re-embedding (#1186)

* first cut at re-embedding route and back-end support

* update and expand config route tests

* add scanpy_umap

* add reembedding to config route parameters

* front-end support for reembedding fetch and UI

* remove unused imports

* add loading state

* save reembedding in reducer state

* improve withColsFrom

* transmit reembed schema to client; pick unique embedding names

* display embeddings

* format

* lint

* spaces, tab size 2

* lint

* test hack for smoke-test race

* back out hack sleep

* add check for backed mode

* add unit test for reembedding

* lint

* hide re-embedding CLI param from help

server/data_anndata/anndata_adaptor.py

@@ -1,17 +1,21 @@
 import warnings
 
 import numpy as np
+import pandas as pd
 from pandas.core.dtypes.dtypes import CategoricalDtype
 import anndata
 from scipy import sparse
 from packaging import version
+from datetime import datetime
+from server_timing import Timing as ServerTiming
 
 from server.data_common.data_adaptor import DataAdaptor
 from server.data_common.fbs.matrix import encode_matrix_fbs
 from server.common.utils import series_to_schema
 from server.common.constants import Axis, MAX_LAYOUTS
-from server.common.errors import PrepareError, DatasetAccessError
+from server.common.errors import PrepareError, DatasetAccessError, FilterError
 from server.common.data_locator import DataLocator
+from server.compute.scanpy import scanpy_umap
 
 anndata_version = version.parse(str(anndata.__version__)).release
 
@@ -261,7 +265,10 @@ class AnndataAdaptor(DataAdaptor):
         return encode_matrix_fbs(df, col_idx=df.columns)
 
     def get_embedding_names(self):
-        """ function:
+        """
+        Return pre-computed embeddings.
+
+        function:
             a) generate list of default layouts
             b) validate layouts are legal.  remove/warn on any that are not
             c) cap total list of layouts at global const MAX_LAYOUTS
@@ -294,6 +301,30 @@ class AnndataAdaptor(DataAdaptor):
         full_embedding = self.data.obsm[f"X_{ename}"]
         return full_embedding[:, 0:dims]
 
+    def compute_embedding(self, method, obsFilter):
+        if Axis.VAR in obsFilter:
+            raise FilterError("Observation filters may not contain variable conditions")
+        if method != "umap":
+            raise NotImplementedError(f"re-embedding method {method} is not available.")
+        try:
+            shape = self.get_shape()
+            obs_mask = self._axis_filter_to_mask(Axis.OBS, obsFilter["obs"], shape[0])
+        except (KeyError, IndexError) as e:
+            raise FilterError(f"Error parsing filter: {e}") from e
+
+        with ServerTiming.time("layout.compute"):
+            X_umap = scanpy_umap(self.data, obs_mask)
+            normalized_layout = DataAdaptor.normalize_embedding(X_umap)
+
+        # Server picks reemedding name, which must not collide with any other
+        # embedding name generated by this backed.
+        name = f"reembed:{method}_{datetime.now().isoformat(timespec='milliseconds')}"
+        dims = [f"{name}_0", f"{name}_1"]
+        df = pd.DataFrame(normalized_layout, columns=dims)
+        fbs = encode_matrix_fbs(df, col_idx=df.columns, row_idx=None)
+        schema = {"name": name, "type": "float32", "dims": dims}
+        return (schema, fbs)
+
     def get_X_array(self, obs_mask=None, var_mask=None):
         if obs_mask is None:
             obs_mask = slice(None)