remove experimental reembedding support (#2301)

* remove experimental reembedding support

* lint

* lint

* add prepare requirements to requirements-dev

* oops, revert accidental deletion of import

* more test modifications

* remove obsolete unit tests

backend/server/app/app.py

@@ -187,11 +187,6 @@ class LayoutObsAPI(Resource):
     def get(self, data_adaptor):
         return common_rest.layout_obs_get(request, data_adaptor)
 
-    @cache_control(no_store=True)
-    @rest_get_data_adaptor
-    def put(self, data_adaptor):
-        return common_rest.layout_obs_put(request, data_adaptor)
-
 
 class GenesetsAPI(Resource):
     @cache_control(public=True, max_age=ONE_WEEK)

backend/server/cli/launch.py

@@ -107,14 +107,6 @@ def config_args(func):
         metavar="<text>",
         help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all.",
     )
-    @click.option(
-        "--experimental-enable-reembedding",
-        is_flag=True,
-        default=DEFAULT_CONFIG.dataset_config.embeddings__enable_reembedding,
-        show_default=False,
-        hidden=True,
-        help="Enable experimental on-demand re-embedding using UMAP. WARNING: may be very slow.",
-    )
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
         return func(*args, **kwargs)
@@ -324,7 +316,6 @@ def launch(
     disable_gene_sets_save,
     backed,
     disable_diffexp,
-    experimental_enable_reembedding,
     config_file,
     dump_default_config,
 ):
@@ -383,7 +374,6 @@ def launch(
             presentation__max_categories=max_category_items,
             presentation__custom_colors=not disable_custom_colors,
             embeddings__names=embedding,
-            embeddings__enable_reembedding=experimental_enable_reembedding,
             diffexp__enable=not disable_diffexp,
             diffexp__lfc_cutoff=diffexp_lfc_cutoff,
         )

backend/server/common/config/client_config.py

@@ -40,7 +40,6 @@ def get_client_config(app_config, data_adaptor):
         "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
         "backed": server_config.adaptor__anndata_adaptor__backed,
         "disable-diffexp": not dataset_config.diffexp__enable,
-        "enable-reembedding": dataset_config.embeddings__enable_reembedding,
         "annotations": False,
         "annotations_file": None,
         "annotations_dir": None,

backend/server/common/config/dataset_config.py

@@ -4,7 +4,6 @@ from os.path import splitext, isdir
 from backend.server.common.annotations.local_file_csv import AnnotationsLocalFile
 from backend.server.common.config.base_config import BaseConfig
 from backend.common.errors import ConfigurationError, AnnotationsError
-from backend.server.compute.scanpy import get_scanpy_module
 from backend.server.data_common.matrix_loader import MatrixDataLoader
 
 
@@ -34,7 +33,6 @@ class DatasetConfig(BaseConfig):
             ]["gene_sets_file"]
 
             self.embeddings__names = default_config["embeddings"]["names"]
-            self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"]
 
             self.diffexp__enable = default_config["diffexp"]["enable"]
             self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
@@ -173,19 +171,6 @@ class DatasetConfig(BaseConfig):
 
     def handle_embeddings(self):
         self.validate_correct_type_of_configuration_attribute("embeddings__names", list)
-        self.validate_correct_type_of_configuration_attribute("embeddings__enable_reembedding", bool)
-
-        server_config = self.app_config.server_config
-        if self.embeddings__enable_reembedding:
-            if server_config.single_dataset__datapath:
-                if server_config.adaptor__anndata_adaptor__backed:
-                    raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
-
-            try:
-                get_scanpy_module()
-            except NotImplementedError:
-                # Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported
-                raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
 
     def handle_diffexp(self, context):
         self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool)

backend/server/common/rest.py

@@ -311,25 +311,6 @@ def layout_obs_get(request, data_adaptor):
         )
 
 
-def layout_obs_put(request, data_adaptor):
-    if not data_adaptor.dataset_config.embeddings__enable_reembedding:
-        return abort(HTTPStatus.NOT_IMPLEMENTED)
-
-    args = request.get_json()
-    filter = args["filter"] if args else None
-    if not filter:
-        return abort_and_log(HTTPStatus.BAD_REQUEST, "obs filter is required")
-    method = args["method"] if args else "umap"
-
-    try:
-        schema = data_adaptor.compute_embedding(method, filter)
-        return make_response(jsonify(schema), HTTPStatus.OK, {"Content-Type": "application/json"})
-    except NotImplementedError as e:
-        return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, str(e))
-    except (ValueError, DisabledFeatureError, FilterError) as e:
-        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
-
-
 def genesets_get(request, data_adaptor):
     preferred_mimetype = request.accept_mimetypes.best_match(["application/json", "text/csv"])
     if preferred_mimetype not in ("application/json", "text/csv"):

backend/server/compute/scanpy.py

@@ -1,53 +0,0 @@
-import importlib
-import numpy as np
-
-"""
-Wrapper for various scanpy modules.  Will raise NotImplementedError if the scanpy
-module is not installed/available
-"""
-
-
-def get_scanpy_module():
-    try:
-        sc = importlib.import_module("scanpy")
-        # Future: we could enforce versions here, eg, lookat sc.__version__
-        return sc
-    except ModuleNotFoundError as e:
-        raise NotImplementedError("Please install scanpy to enable UMAP re-embedding") from e
-    except Exception as e:
-        # will capture other ImportError corner cases
-        raise NotImplementedError() from e
-
-
-def scanpy_umap(adata, obs_mask=None, pca_options={}, neighbors_options={}, umap_options={}):
-    """
-    Given adata and an obs mask, return a new embedding for adata[obs_mask, :]
-    as an ndarray of shape (len(obs_mask), N), where N>=2.
-
-    Do NOT mutate adata.
-    """
-
-    # backed mode is incompatible with the current implementation
-    if adata.isbacked:
-        raise NotImplementedError("Backed mode is incompatible with re-embedding")
-
-    # safely get scanpy module, which may not be present.
-    sc = get_scanpy_module()
-
-    # https://github.com/theislab/anndata/issues/311
-    obs_mask = slice(None) if obs_mask is None else obs_mask
-    adata = adata[obs_mask, :].copy()
-
-    for k in list(adata.obsm.keys()):
-        del adata.obsm[k]
-    for k in list(adata.uns.keys()):
-        del adata.uns[k]
-
-    sc.pp.pca(adata, zero_center=None, n_comps=min(adata.n_vars - 1, 50), **pca_options)
-    sc.pp.neighbors(adata, **neighbors_options)
-    sc.tl.umap(adata, **umap_options)
-
-    umap = adata.obsm["X_umap"]
-    result = np.full((obs_mask.shape[0], umap.shape[1]), np.NaN)
-    result[obs_mask] = umap
-    return result

backend/server/data_anndata/anndata_adaptor.py

@@ -1,20 +1,17 @@
 import warnings
-from datetime import datetime
 
 import anndata
 import numpy as np
 from packaging import version
 from pandas.core.dtypes.dtypes import CategoricalDtype
 from scipy import sparse
-from server_timing import Timing as ServerTiming
 
 import backend.common.compute.diffexp_generic as diffexp_generic
 from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
 from backend.common.constants import Axis, MAX_LAYOUTS
 from backend.server.common.corpora import corpora_get_props_from_anndata
-from backend.common.errors import PrepareError, DatasetAccessError, FilterError
+from backend.common.errors import PrepareError, DatasetAccessError
 from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
-from backend.server.compute.scanpy import scanpy_umap
 from backend.server.data_common.data_adaptor import DataAdaptor
 from backend.common.fbs.matrix import encode_matrix_fbs
 
@@ -301,28 +298,6 @@ class AnndataAdaptor(DataAdaptor):
         full_embedding = self.data.obsm[f"X_{ename}"]
         return full_embedding[:, 0:dims]
 
-    def compute_embedding(self, method, obsFilter):
-        if Axis.VAR in obsFilter:
-            raise FilterError("Observation filters may not contain variable conditions")
-        if method != "umap":
-            raise NotImplementedError(f"re-embedding method {method} is not available.")
-        try:
-            shape = self.get_shape()
-            obs_mask = self._axis_filter_to_mask(Axis.OBS, obsFilter["obs"], shape[0])
-        except (KeyError, IndexError):
-            raise FilterError("Error parsing filter")
-        with ServerTiming.time("layout.compute"):
-            X_umap = scanpy_umap(self.data, obs_mask)
-
-        # Server picks reemedding name, which must not collide with any other
-        # embedding name generated by this backend.
-        name = f"reembed:{method}_{datetime.now().isoformat(timespec='milliseconds')}"
-        dims = [f"{name}_0", f"{name}_1"]
-        layout_schema = {"name": name, "type": "float32", "dims": dims}
-        self.schema["layout"]["obs"].append(layout_schema)
-        self.data.obsm[f"X_{name}"] = X_umap
-        return layout_schema
-
     def compute_diffexp_ttest(self, maskA, maskB, top_n=None, lfc_cutoff=None):
         if top_n is None:
             top_n = self.dataset_config.diffexp__top_n

backend/server/data_common/data_adaptor.py

@@ -66,11 +66,6 @@ class DataAdaptor(metaclass=ABCMeta):
         """return an numpy array for the given pre-computed embedding name."""
         pass
 
-    @abstractmethod
-    def compute_embedding(self, method, filter):
-        """compute a new embedding on the specified obs subset, and return the embedding schema."""
-        pass
-
     @abstractmethod
     def get_X_array(self, obs_mask=None, var_mask=None):
         """return the X array, possibly filtered by obs_mask or var_mask.

backend/server/default_config.py

@@ -71,7 +71,6 @@ dataset:
 
   embeddings:
     names : []
-    enable_reembedding: false
 
   diffexp:
     enable: true

backend/server/requirements-dev.txt

@@ -8,3 +8,4 @@ pytest>=3.6.3
 python-jose>=3.2.0
 twine>=1.12.1
 -r requirements.txt
+-r requirements-prepare.txt

backend/server/requirements-prepare.txt

@@ -1,2 +1,4 @@
 python-igraph>=0.8
 louvain>=0.6
+scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here
+umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn

backend/server/requirements.txt

@@ -21,5 +21,3 @@ PyYAML>=5.4  # CVE-2020-14343
 scipy>=1.4
 requests>=2.22.0
 s3fs==0.4.2
-scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here
-umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn