remove experimental reembedding support (#2301)

* remove experimental reembedding support

* lint

* lint

* add prepare requirements to requirements-dev

* oops, revert accidental deletion of import

* more test modifications

* remove obsolete unit tests

backend/czi_hosted/app/app.py

@@ -312,11 +312,6 @@ class LayoutObsAPI(DatasetResource):
     def get(self, data_adaptor):
         return common_rest.layout_obs_get(request, data_adaptor)
 
-    @cache_control(no_store=True)
-    @rest_get_data_adaptor
-    def put(self, data_adaptor):
-        return common_rest.layout_obs_put(request, data_adaptor)
-
 
 class GenesetsAPI(DatasetResource):
     @cache_control(public=True, max_age=ONE_WEEK)

backend/czi_hosted/cli/launch.py

@@ -90,14 +90,6 @@ def config_args(func):
         metavar="<text>",
         help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all.",
     )
-    @click.option(
-        "--experimental-enable-reembedding",
-        is_flag=True,
-        default=DEFAULT_CONFIG.default_dataset_config.embeddings__enable_reembedding,
-        show_default=False,
-        hidden=True,
-        help="Enable experimental on-demand re-embedding using UMAP. WARNING: may be very slow.",
-    )
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
         return func(*args, **kwargs)
@@ -314,7 +306,6 @@ def launch(
     annotations_dir,
     backed,
     disable_diffexp,
-    experimental_enable_reembedding,
     config_file,
     dump_default_config,
 ):
@@ -376,7 +367,6 @@ def launch(
             presentation__max_categories=max_category_items,
             presentation__custom_colors=not disable_custom_colors,
             embeddings__names=embedding,
-            embeddings__enable_reembedding=experimental_enable_reembedding,
             diffexp__enable=not disable_diffexp,
             diffexp__lfc_cutoff=diffexp_lfc_cutoff,
         )

backend/czi_hosted/common/config/client_config.py

@@ -40,7 +40,6 @@ def get_client_config(app_config, data_adaptor):
         "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
         "backed": server_config.adaptor__anndata_adaptor__backed,
         "disable-diffexp": not dataset_config.diffexp__enable,
-        "enable-reembedding": dataset_config.embeddings__enable_reembedding,
         "annotations": False,
         "annotations_file": None,
         "annotations_dir": None,

backend/czi_hosted/common/config/dataset_config.py

@@ -6,8 +6,6 @@ from backend.czi_hosted.common.annotations.hosted_tiledb import AnnotationsHoste
 from backend.czi_hosted.common.annotations.local_file_csv import AnnotationsLocalFile
 from backend.czi_hosted.common.config.base_config import BaseConfig
 from backend.common.errors import ConfigurationError
-from backend.czi_hosted.compute.scanpy import get_scanpy_module
-from backend.czi_hosted.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
 from backend.czi_hosted.db.db_utils import DbUtils
 
 
@@ -41,7 +39,6 @@ class DatasetConfig(BaseConfig):
             ]["hosted_file_directory"]
 
             self.embeddings__names = default_config["embeddings"]["names"]
-            self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"]
 
             self.diffexp__enable = default_config["diffexp"]["enable"]
             self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
@@ -186,24 +183,6 @@ class DatasetConfig(BaseConfig):
 
     def handle_embeddings(self):
         self.validate_correct_type_of_configuration_attribute("embeddings__names", list)
-        self.validate_correct_type_of_configuration_attribute("embeddings__enable_reembedding", bool)
-
-        server_config = self.app_config.server_config
-        if self.embeddings__enable_reembedding:
-            if server_config.single_dataset__datapath:
-                matrix_data_loader = MatrixDataLoader(
-                    server_config.single_dataset__datapath, app_config=self.app_config
-                )
-                if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD:
-                    raise ConfigurationError("enable-reembedding is only supported with H5AD files.")
-                if server_config.adaptor__anndata_adaptor__backed:
-                    raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
-
-            try:
-                get_scanpy_module()
-            except NotImplementedError:
-                # Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported
-                raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
 
     def handle_diffexp(self, context):
         self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool)

backend/czi_hosted/common/rest.py

@@ -312,25 +312,6 @@ def layout_obs_get(request, data_adaptor):
         )
 
 
-def layout_obs_put(request, data_adaptor):
-    if not data_adaptor.dataset_config.embeddings__enable_reembedding:
-        return abort(HTTPStatus.NOT_IMPLEMENTED)
-
-    args = request.get_json()
-    filter = args["filter"] if args else None
-    if not filter:
-        return abort_and_log(HTTPStatus.BAD_REQUEST, "obs filter is required")
-    method = args["method"] if args else "umap"
-
-    try:
-        schema = data_adaptor.compute_embedding(method, filter)
-        return make_response(jsonify(schema), HTTPStatus.OK, {"Content-Type": "application/json"})
-    except NotImplementedError as e:
-        return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, str(e))
-    except (ValueError, DisabledFeatureError, FilterError) as e:
-        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
-
-
 def genesets_get(request, data_adaptor):
     preferred_mimetype = request.accept_mimetypes.best_match(["application/json", "text/csv"])
     if preferred_mimetype not in ("application/json", "text/csv"):

backend/czi_hosted/compute/scanpy.py

@@ -1,53 +0,0 @@
-import importlib
-import numpy as np
-
-"""
-Wrapper for various scanpy modules.  Will raise NotImplementedError if the scanpy
-module is not installed/available
-"""
-
-
-def get_scanpy_module():
-    try:
-        sc = importlib.import_module("scanpy")
-        # Future: we could enforce versions here, eg, lookat sc.__version__
-        return sc
-    except ModuleNotFoundError as e:
-        raise NotImplementedError("Please install scanpy to enable UMAP re-embedding") from e
-    except Exception as e:
-        # will capture other ImportError corner cases
-        raise NotImplementedError() from e
-
-
-def scanpy_umap(adata, obs_mask=None, pca_options={}, neighbors_options={}, umap_options={}):
-    """
-    Given adata and an obs mask, return a new embedding for adata[obs_mask, :]
-    as an ndarray of shape (len(obs_mask), N), where N>=2.
-
-    Do NOT mutate adata.
-    """
-
-    # backed mode is incompatible with the current implementation
-    if adata.isbacked:
-        raise NotImplementedError("Backed mode is incompatible with re-embedding")
-
-    # safely get scanpy module, which may not be present.
-    sc = get_scanpy_module()
-
-    # https://github.com/theislab/anndata/issues/311
-    obs_mask = slice(None) if obs_mask is None else obs_mask
-    adata = adata[obs_mask, :].copy()
-
-    for k in list(adata.obsm.keys()):
-        del adata.obsm[k]
-    for k in list(adata.uns.keys()):
-        del adata.uns[k]
-
-    sc.pp.pca(adata, zero_center=None, n_comps=min(adata.n_vars - 1, 50), **pca_options)
-    sc.pp.neighbors(adata, **neighbors_options)
-    sc.tl.umap(adata, **umap_options)
-
-    umap = adata.obsm["X_umap"]
-    result = np.full((obs_mask.shape[0], umap.shape[1]), np.NaN)
-    result[obs_mask] = umap
-    return result

backend/czi_hosted/data_anndata/anndata_adaptor.py

@@ -1,20 +1,17 @@
 import warnings
-from datetime import datetime
 
 import anndata
 import numpy as np
 from packaging import version
 from pandas.core.dtypes.dtypes import CategoricalDtype
 from scipy import sparse
-from server_timing import Timing as ServerTiming
 
 import backend.common.compute.diffexp_generic as diffexp_generic
 from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
 from backend.common.constants import Axis, MAX_LAYOUTS
 from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata
-from backend.common.errors import PrepareError, DatasetAccessError, FilterError
+from backend.common.errors import PrepareError, DatasetAccessError
 from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
-from backend.czi_hosted.compute.scanpy import scanpy_umap
 from backend.czi_hosted.data_common.data_adaptor import DataAdaptor
 from backend.common.fbs.matrix import encode_matrix_fbs
 
@@ -301,28 +298,6 @@ class AnndataAdaptor(DataAdaptor):
         full_embedding = self.data.obsm[f"X_{ename}"]
         return full_embedding[:, 0:dims]
 
-    def compute_embedding(self, method, obsFilter):
-        if Axis.VAR in obsFilter:
-            raise FilterError("Observation filters may not contain variable conditions")
-        if method != "umap":
-            raise NotImplementedError(f"re-embedding method {method} is not available.")
-        try:
-            shape = self.get_shape()
-            obs_mask = self._axis_filter_to_mask(Axis.OBS, obsFilter["obs"], shape[0])
-        except (KeyError, IndexError):
-            raise FilterError("Error parsing filter")
-        with ServerTiming.time("layout.compute"):
-            X_umap = scanpy_umap(self.data, obs_mask)
-
-        # Server picks reemedding name, which must not collide with any other
-        # embedding name generated by this backend.
-        name = f"reembed:{method}_{datetime.now().isoformat(timespec='milliseconds')}"
-        dims = [f"{name}_0", f"{name}_1"]
-        layout_schema = {"name": name, "type": "float32", "dims": dims}
-        self.schema["layout"]["obs"].append(layout_schema)
-        self.data.obsm[f"X_{name}"] = X_umap
-        return layout_schema
-
     def compute_diffexp_ttest(self, maskA, maskB, top_n=None, lfc_cutoff=None):
         if top_n is None:
             top_n = self.dataset_config.diffexp__top_n

backend/czi_hosted/data_common/data_adaptor.py

@@ -71,11 +71,6 @@ class DataAdaptor(metaclass=ABCMeta):
         """return an numpy array for the given pre-computed embedding name."""
         pass
 
-    @abstractmethod
-    def compute_embedding(self, method, filter):
-        """compute a new embedding on the specified obs subset, and return the embedding schema. """
-        pass
-
     @abstractmethod
     def get_X_array(self, obs_mask=None, var_mask=None):
         """return the X array, possibly filtered by obs_mask or var_mask.

backend/czi_hosted/data_cxg/cxg_adaptor.py

@@ -199,9 +199,6 @@ class CxgAdaptor(DataAdaptor):
         array = self.open_array(f"emb/{ename}")
         return array[:, 0:dims]
 
-    def compute_embedding(self, method, filter):
-        raise NotImplementedError("CXG does not yet support re-embedding")
-
     def compute_diffexp_ttest(self, maskA, maskB, top_n=None, lfc_cutoff=None):
         if top_n is None:
             top_n = self.dataset_config.diffexp__top_n

backend/czi_hosted/default_config.py

@@ -197,7 +197,6 @@ dataset:
 
   embeddings:
     names : []
-    enable_reembedding: false
 
   diffexp:
     enable: true

backend/czi_hosted/eb/app.py

@@ -164,7 +164,6 @@ try:
         app_config.update_server_config(multi_dataset__dataroot=dataroot)
 
     # overwrite configuration for the eb app
-    app_config.update_default_dataset_config(embeddings__enable_reembedding=False,)
     app_config.update_server_config(multi_dataset__allowed_matrix_types=["cxg"],)
 
     # complete config

backend/czi_hosted/requirements-dev.txt

@@ -8,4 +8,5 @@ pytest>=3.6.3
 python-jose>=3.2.0
 twine>=1.12.1
 -r requirements.txt
+-r requirements-prepare.txt
 rsa>=4.7 # not directly required, pinned by Snyk to avoid a vulnerability

backend/czi_hosted/requirements-prepare.txt

@@ -1,2 +1,4 @@
 python-igraph
 louvain>=0.6
+scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here
+umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn

backend/czi_hosted/requirements.txt

@@ -21,6 +21,4 @@ scipy>=1.0
 requests>=2.22.0
 tiledb>=0.5.9,>=0.6.2,!=0.7.2, !=0.8.6
 s3fs==0.4.2
-scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here
 sqlalchemy>=1.3.18
-umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn

backend/server/app/app.py

@@ -187,11 +187,6 @@ class LayoutObsAPI(Resource):
     def get(self, data_adaptor):
         return common_rest.layout_obs_get(request, data_adaptor)
 
-    @cache_control(no_store=True)
-    @rest_get_data_adaptor
-    def put(self, data_adaptor):
-        return common_rest.layout_obs_put(request, data_adaptor)
-
 
 class GenesetsAPI(Resource):
     @cache_control(public=True, max_age=ONE_WEEK)

backend/server/cli/launch.py

@@ -107,14 +107,6 @@ def config_args(func):
         metavar="<text>",
         help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all.",
     )
-    @click.option(
-        "--experimental-enable-reembedding",
-        is_flag=True,
-        default=DEFAULT_CONFIG.dataset_config.embeddings__enable_reembedding,
-        show_default=False,
-        hidden=True,
-        help="Enable experimental on-demand re-embedding using UMAP. WARNING: may be very slow.",
-    )
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
         return func(*args, **kwargs)
@@ -324,7 +316,6 @@ def launch(
     disable_gene_sets_save,
     backed,
     disable_diffexp,
-    experimental_enable_reembedding,
     config_file,
     dump_default_config,
 ):
@@ -383,7 +374,6 @@ def launch(
             presentation__max_categories=max_category_items,
             presentation__custom_colors=not disable_custom_colors,
             embeddings__names=embedding,
-            embeddings__enable_reembedding=experimental_enable_reembedding,
             diffexp__enable=not disable_diffexp,
             diffexp__lfc_cutoff=diffexp_lfc_cutoff,
         )

backend/server/common/config/client_config.py

@@ -40,7 +40,6 @@ def get_client_config(app_config, data_adaptor):
         "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
         "backed": server_config.adaptor__anndata_adaptor__backed,
         "disable-diffexp": not dataset_config.diffexp__enable,
-        "enable-reembedding": dataset_config.embeddings__enable_reembedding,
         "annotations": False,
         "annotations_file": None,
         "annotations_dir": None,

backend/server/common/config/dataset_config.py

@@ -4,7 +4,6 @@ from os.path import splitext, isdir
 from backend.server.common.annotations.local_file_csv import AnnotationsLocalFile
 from backend.server.common.config.base_config import BaseConfig
 from backend.common.errors import ConfigurationError, AnnotationsError
-from backend.server.compute.scanpy import get_scanpy_module
 from backend.server.data_common.matrix_loader import MatrixDataLoader
 
 
@@ -34,7 +33,6 @@ class DatasetConfig(BaseConfig):
             ]["gene_sets_file"]
 
             self.embeddings__names = default_config["embeddings"]["names"]
-            self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"]
 
             self.diffexp__enable = default_config["diffexp"]["enable"]
             self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
@@ -173,19 +171,6 @@ class DatasetConfig(BaseConfig):
 
     def handle_embeddings(self):
         self.validate_correct_type_of_configuration_attribute("embeddings__names", list)
-        self.validate_correct_type_of_configuration_attribute("embeddings__enable_reembedding", bool)
-
-        server_config = self.app_config.server_config
-        if self.embeddings__enable_reembedding:
-            if server_config.single_dataset__datapath:
-                if server_config.adaptor__anndata_adaptor__backed:
-                    raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
-
-            try:
-                get_scanpy_module()
-            except NotImplementedError:
-                # Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported
-                raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
 
     def handle_diffexp(self, context):
         self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool)

backend/server/common/rest.py

@@ -311,25 +311,6 @@ def layout_obs_get(request, data_adaptor):
         )
 
 
-def layout_obs_put(request, data_adaptor):
-    if not data_adaptor.dataset_config.embeddings__enable_reembedding:
-        return abort(HTTPStatus.NOT_IMPLEMENTED)
-
-    args = request.get_json()
-    filter = args["filter"] if args else None
-    if not filter:
-        return abort_and_log(HTTPStatus.BAD_REQUEST, "obs filter is required")
-    method = args["method"] if args else "umap"
-
-    try:
-        schema = data_adaptor.compute_embedding(method, filter)
-        return make_response(jsonify(schema), HTTPStatus.OK, {"Content-Type": "application/json"})
-    except NotImplementedError as e:
-        return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, str(e))
-    except (ValueError, DisabledFeatureError, FilterError) as e:
-        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
-
-
 def genesets_get(request, data_adaptor):
     preferred_mimetype = request.accept_mimetypes.best_match(["application/json", "text/csv"])
     if preferred_mimetype not in ("application/json", "text/csv"):

backend/server/compute/scanpy.py

@@ -1,53 +0,0 @@
-import importlib
-import numpy as np
-
-"""
-Wrapper for various scanpy modules.  Will raise NotImplementedError if the scanpy
-module is not installed/available
-"""
-
-
-def get_scanpy_module():
-    try:
-        sc = importlib.import_module("scanpy")
-        # Future: we could enforce versions here, eg, lookat sc.__version__
-        return sc
-    except ModuleNotFoundError as e:
-        raise NotImplementedError("Please install scanpy to enable UMAP re-embedding") from e
-    except Exception as e:
-        # will capture other ImportError corner cases
-        raise NotImplementedError() from e
-
-
-def scanpy_umap(adata, obs_mask=None, pca_options={}, neighbors_options={}, umap_options={}):
-    """
-    Given adata and an obs mask, return a new embedding for adata[obs_mask, :]
-    as an ndarray of shape (len(obs_mask), N), where N>=2.
-
-    Do NOT mutate adata.
-    """
-
-    # backed mode is incompatible with the current implementation
-    if adata.isbacked:
-        raise NotImplementedError("Backed mode is incompatible with re-embedding")
-
-    # safely get scanpy module, which may not be present.
-    sc = get_scanpy_module()
-
-    # https://github.com/theislab/anndata/issues/311
-    obs_mask = slice(None) if obs_mask is None else obs_mask
-    adata = adata[obs_mask, :].copy()
-
-    for k in list(adata.obsm.keys()):
-        del adata.obsm[k]
-    for k in list(adata.uns.keys()):
-        del adata.uns[k]
-
-    sc.pp.pca(adata, zero_center=None, n_comps=min(adata.n_vars - 1, 50), **pca_options)
-    sc.pp.neighbors(adata, **neighbors_options)
-    sc.tl.umap(adata, **umap_options)
-
-    umap = adata.obsm["X_umap"]
-    result = np.full((obs_mask.shape[0], umap.shape[1]), np.NaN)
-    result[obs_mask] = umap
-    return result

backend/server/data_anndata/anndata_adaptor.py

@@ -1,20 +1,17 @@
 import warnings
-from datetime import datetime
 
 import anndata
 import numpy as np
 from packaging import version
 from pandas.core.dtypes.dtypes import CategoricalDtype
 from scipy import sparse
-from server_timing import Timing as ServerTiming
 
 import backend.common.compute.diffexp_generic as diffexp_generic
 from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
 from backend.common.constants import Axis, MAX_LAYOUTS
 from backend.server.common.corpora import corpora_get_props_from_anndata
-from backend.common.errors import PrepareError, DatasetAccessError, FilterError
+from backend.common.errors import PrepareError, DatasetAccessError
 from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
-from backend.server.compute.scanpy import scanpy_umap
 from backend.server.data_common.data_adaptor import DataAdaptor
 from backend.common.fbs.matrix import encode_matrix_fbs
 
@@ -301,28 +298,6 @@ class AnndataAdaptor(DataAdaptor):
         full_embedding = self.data.obsm[f"X_{ename}"]
         return full_embedding[:, 0:dims]
 
-    def compute_embedding(self, method, obsFilter):
-        if Axis.VAR in obsFilter:
-            raise FilterError("Observation filters may not contain variable conditions")
-        if method != "umap":
-            raise NotImplementedError(f"re-embedding method {method} is not available.")
-        try:
-            shape = self.get_shape()
-            obs_mask = self._axis_filter_to_mask(Axis.OBS, obsFilter["obs"], shape[0])
-        except (KeyError, IndexError):
-            raise FilterError("Error parsing filter")
-        with ServerTiming.time("layout.compute"):
-            X_umap = scanpy_umap(self.data, obs_mask)
-
-        # Server picks reemedding name, which must not collide with any other
-        # embedding name generated by this backend.
-        name = f"reembed:{method}_{datetime.now().isoformat(timespec='milliseconds')}"
-        dims = [f"{name}_0", f"{name}_1"]
-        layout_schema = {"name": name, "type": "float32", "dims": dims}
-        self.schema["layout"]["obs"].append(layout_schema)
-        self.data.obsm[f"X_{name}"] = X_umap
-        return layout_schema
-
     def compute_diffexp_ttest(self, maskA, maskB, top_n=None, lfc_cutoff=None):
         if top_n is None:
             top_n = self.dataset_config.diffexp__top_n

backend/server/data_common/data_adaptor.py

@@ -66,11 +66,6 @@ class DataAdaptor(metaclass=ABCMeta):
         """return an numpy array for the given pre-computed embedding name."""
         pass
 
-    @abstractmethod
-    def compute_embedding(self, method, filter):
-        """compute a new embedding on the specified obs subset, and return the embedding schema."""
-        pass
-
     @abstractmethod
     def get_X_array(self, obs_mask=None, var_mask=None):
         """return the X array, possibly filtered by obs_mask or var_mask.

backend/server/default_config.py

@@ -71,7 +71,6 @@ dataset:
 
   embeddings:
     names : []
-    enable_reembedding: false
 
   diffexp:
     enable: true

backend/server/requirements-dev.txt

@@ -8,3 +8,4 @@ pytest>=3.6.3
 python-jose>=3.2.0
 twine>=1.12.1
 -r requirements.txt
+-r requirements-prepare.txt

backend/server/requirements-prepare.txt

@@ -1,2 +1,4 @@
 python-igraph>=0.8
 louvain>=0.6
+scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here
+umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn

backend/server/requirements.txt

@@ -21,5 +21,3 @@ PyYAML>=5.4  # CVE-2020-14343
 scipy>=1.4
 requests>=2.22.0
 s3fs==0.4.2
-scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here
-umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn

backend/test/fixtures/czi_hosted_dataset_config_outline.py

@@ -25,7 +25,6 @@ dataset:
 
   embeddings:
     names: {embedding_names}
-    enable_reembedding: {enable_reembedding}
 
   diffexp:
     enable: {enable_difexp}

backend/test/fixtures/dataset_config_outline.py

@@ -22,7 +22,6 @@ dataset:
 
   embeddings:
     names: {embedding_names}
-    enable_reembedding: {enable_reembedding}
 
   diffexp:
     enable: {enable_difexp}

backend/test/test_czi_hosted/unit/__init__.py

@@ -170,7 +170,6 @@ class BaseTest(unittest.TestCase):
                 multi_dataset__index=True,
                 multi_dataset__allowed_matrix_types=["cxg"]
             )
-            app_config.update_default_dataset_config(embeddings__enable_reembedding=False, )
         app_config.complete_config(logging.info)
 
         app = TestServer(app_config).app

backend/test/test_czi_hosted/unit/common/config/__init__.py

@@ -125,7 +125,6 @@ class ConfigTests(BaseTest):
         local_file_csv_directory="null",
         local_file_csv_file="null",
         embedding_names=[],
-        enable_reembedding="false",
         enable_difexp="true",
         lfc_cutoff=0.01,
         top_n=10,
@@ -192,7 +191,6 @@ class ConfigTests(BaseTest):
             local_file_csv_directory=local_file_csv_directory,
             local_file_csv_file=local_file_csv_file,
             embedding_names=embedding_names,
-            enable_reembedding=enable_reembedding,
             enable_difexp=enable_difexp,
             lfc_cutoff=lfc_cutoff,
             top_n=top_n,
@@ -228,7 +226,6 @@ class ConfigTests(BaseTest):
         local_file_csv_directory="null",
         local_file_csv_file="null",
         embedding_names=[],
-        enable_reembedding="false",
         enable_difexp="true",
         lfc_cutoff=0.01,
         top_n=10,

backend/test/test_czi_hosted/unit/common/config/test_dataset_config.py

@@ -49,7 +49,7 @@ class TestDatasetConfig(ConfigTests):
     def test_complete_config_checks_all_attr(self, mock_check_attrs):
         mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
         self.dataset_config.complete_config(self.context)
-        self.assertEqual(mock_check_attrs.call_count, 19)
+        self.assertEqual(mock_check_attrs.call_count, 18)
 
     def test_app_sets_script_vars(self):
         config = self.get_config(scripts=["path/to/script"])
@@ -130,20 +130,6 @@ class TestDatasetConfig(ConfigTests):
         cwd = os.getcwd()
         self.assertEqual(config.default_dataset_config.user_annotations._get_output_dir(), cwd)
 
-    def test_handle_embeddings__checks_data_file_types(self):
-        file_name = self.custom_app_config(
-            embedding_names=["name1", "name2"],
-            enable_reembedding="true",
-            dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
-            anndata_backed="true",
-            config_file_name=self.config_file_name,
-        )
-        config = AppConfig()
-        config.update_from_config_file(file_name)
-        config.server_config.complete_config(self.context)
-        with self.assertRaises(ConfigurationError):
-            config.default_dataset_config.handle_embeddings()
-
     def test_handle_diffexp__raises_warning_for_large_datasets(self):
         config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15)
         config.server_config.complete_config(self.context)

backend/test/test_czi_hosted/unit/common/test_api.py

@@ -69,35 +69,6 @@ class EndPoints(BaseTest):
         self.assertIsNone(df["row_idx"])
         self.assertEqual(len(df["columns"]), df["n_cols"])
 
-    def test_put_layout_fbs(self):
-        # first check that re-embedding is turned on
-        self.app.auth.get_user_id = lambda : "123"
-        result = self.client.get(f"{self.TEST_URL_BASE}config")
-        config_data = json.loads(result.data)
-        re_embed = config_data["config"]["parameters"]["enable-reembedding"]
-        if not re_embed:
-            return
-        # attempt to reembed with umap over 100 cells.
-        endpoint = "layout/obs"
-        url = f"{self.TEST_URL_BASE}{endpoint}"
-        data = {}
-        data["filter"] = {}
-        data["filter"]["obs"] = {}
-        data["filter"]["obs"]["index"] = list(range(100))
-        data["method"] = "umap"
-        result = self.client.put(url, json=data)
-
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        result_data = result.json()
-        self.assertIsInstance(result_data, dict)
-        self.assertEqual(result_data["type"], "float32")
-        self.assertTrue(result_data["name"].startswith("reembed:umap_"))
-        self.assertIsInstance(result_data["dims"], list)
-        self.assertEqual(len(result_data["dims"]), 2)
-        dims = result_data["dims"]
-        self.assertTrue(dims[0].startswith("reembed:umap_") and dims[0].endswith("_0"))
-        self.assertTrue(dims[1].startswith("reembed:umap_") and dims[1].endswith("_1"))
-
     def test_bad_filter(self):
         endpoint = "data/var"
         url = f"{self.TEST_URL_BASE}{endpoint}"
@@ -421,7 +392,7 @@ class EndPointsCxg(EndPoints):
     @classmethod
     def setUpClass(cls):
         app_config = AppConfig()
-        app_config.update_default_dataset_config(embeddings__enable_reembedding=True, user_annotations__enable=False)
+        app_config.update_default_dataset_config(user_annotations__enable=False)
 
     def test_get_genesets_json(self):
         self.app.auth.is_user_authenticated = lambda: True

backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor.py

@@ -196,39 +196,3 @@ class AdaptorTest(unittest.TestCase):
         self.assertEqual(data["n_rows"], 2638)
         self.assertEqual(data["n_cols"], 3)
         self.assertTrue((data["col_idx"] == [15, 1818, 1837]).all())
-
-    def test_compute_embedding(self):
-        filter = {"obs": {"index": [[0, 100]]}}
-
-        # Verify that we correctly handle the case where we lack scanpy
-        import unittest.mock
-
-        with unittest.mock.patch.dict(sys.modules, {"scanpy": None}):
-            with self.assertRaises(NotImplementedError):
-                self.data.compute_embedding("umap", filter)
-
-        # if we happen to have scanpy, test the full API, else punt
-        import importlib
-
-        scanpy_spec = importlib.util.find_spec("scanpy")
-        if scanpy_spec is None:
-            print("Skipping compute_embedding test as ScanPy not installed")
-            return
-
-        # this feature is unsupported in backed mode, and we expect an error
-        if self.data.data.isbacked:
-            with self.assertRaises(NotImplementedError):
-                self.data.compute_embedding("umap", filter)
-            return
-
-        schema = self.data.compute_embedding("umap", filter)
-
-        self.assertIsInstance(schema["name"], str)
-        name = schema["name"]
-        self.assertEqual(schema["type"], "float32")
-        self.assertEqual(schema["dims"], [f"{name}_0", f"{name}_1"])
-
-        emb = self.data.data.obsm[f"X_{name}"]
-        self.assertEqual(emb.shape, (2638, 2))
-        self.assertTrue(np.isfinite(emb[0:100]).all())
-        self.assertTrue(np.isnan(emb[100:]).all())

backend/test/test_server/unit/common/config/__init__.py

@@ -97,7 +97,6 @@ class ConfigTests(unittest.TestCase):
         local_file_csv_gene_sets_file="null",
         gene_sets_readonly="false",
         embedding_names=[],
-        enable_reembedding="false",
         enable_difexp="true",
         lfc_cutoff=0.01,
         top_n=10,
@@ -148,7 +147,6 @@ class ConfigTests(unittest.TestCase):
             local_file_csv_gene_sets_file=local_file_csv_gene_sets_file,
             gene_sets_readonly=gene_sets_readonly,
             embedding_names=embedding_names,
-            enable_reembedding=enable_reembedding,
             enable_difexp=enable_difexp,
             lfc_cutoff=lfc_cutoff,
             top_n=top_n,
@@ -184,7 +182,6 @@ class ConfigTests(unittest.TestCase):
         local_file_csv_gene_sets_file="null",
         gene_sets_readonly="false",
         embedding_names=[],
-        enable_reembedding="false",
         enable_difexp="true",
         lfc_cutoff=0.01,
         top_n=10,

backend/test/test_server/unit/common/config/test_dataset_config.py

@@ -46,7 +46,7 @@ class TestDatasetConfig(ConfigTests):
         mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
         self.dataset_config.complete_config(self.context)
         self.assertIsNotNone(self.config.server_config.data_adaptor)
-        self.assertEqual(mock_check_attrs.call_count, 17)
+        self.assertEqual(mock_check_attrs.call_count, 16)
 
     def test_app_sets_script_vars(self):
         config = self.get_config(scripts=["path/to/script"])
@@ -106,20 +106,6 @@ class TestDatasetConfig(ConfigTests):
         cwd = os.getcwd()
         self.assertEqual(config.dataset_config.user_annotations._get_output_dir(), cwd)
 
-    def test_handle_embeddings__checks_data_file_types(self):
-        file_name = self.custom_app_config(
-            embedding_names=["name1", "name2"],
-            enable_reembedding="true",
-            dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
-            anndata_backed="true",
-            config_file_name=self.config_file_name,
-        )
-        config = AppConfig()
-        config.update_from_config_file(file_name)
-        config.server_config.complete_config(self.context)
-        with self.assertRaises(ConfigurationError):
-            config.dataset_config.handle_embeddings()
-
     def test_handle_diffexp__raises_warning_for_large_datasets(self):
         config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15)
         config.server_config.complete_config(self.context)

backend/test/test_server/unit/common/test_api.py

@@ -73,34 +73,6 @@ class EndPoints(object):
         self.assertIsNone(df["row_idx"])
         self.assertEqual(len(df["columns"]), df["n_cols"])
 
-    def test_put_layout_fbs(self):
-        # first check that re-embedding is turned on
-        result = self.session.get(f"{self.URL_BASE}config")
-        config_data = result.json()
-        re_embed = config_data["config"]["parameters"]["enable-reembedding"]
-        if not re_embed:
-            return
-        # attempt to reembed with umap over 100 cells.
-        endpoint = "layout/obs"
-        url = f"{self.URL_BASE}{endpoint}"
-        data = {}
-        data["filter"] = {}
-        data["filter"]["obs"] = {}
-        data["filter"]["obs"]["index"] = list(range(100))
-        data["method"] = "umap"
-        result = self.session.put(url, json=data)
-
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        result_data = result.json()
-        self.assertIsInstance(result_data, dict)
-        self.assertEqual(result_data["type"], "float32")
-        self.assertTrue(result_data["name"].startswith("reembed:umap_"))
-        self.assertIsInstance(result_data["dims"], list)
-        self.assertEqual(len(result_data["dims"]), 2)
-        dims = result_data["dims"]
-        self.assertTrue(dims[0].startswith("reembed:umap_") and dims[0].endswith("_0"))
-        self.assertTrue(dims[1].startswith("reembed:umap_") and dims[1].endswith("_1"))
-
     def test_bad_filter(self):
         endpoint = "data/var"
         url = f"{self.URL_BASE}{endpoint}"
@@ -389,7 +361,6 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
                 f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
                 "--disable-annotations",
                 "--disable-gene-sets-save",
-                "--experimental-enable-reembedding",
             ],
         )
 

backend/test/test_server/unit/data_anndata/test_anndata_adaptor.py

@@ -198,39 +198,3 @@ class AdaptorTest(unittest.TestCase):
         self.assertEqual(data["n_rows"], 2638)
         self.assertEqual(data["n_cols"], 3)
         self.assertTrue((data["col_idx"] == [15, 1818, 1837]).all())
-
-    def test_compute_embedding(self):
-        filter = {"obs": {"index": [[0, 100]]}}
-
-        # Verify that we correctly handle the case where we lack scanpy
-        import unittest.mock
-
-        with unittest.mock.patch.dict(sys.modules, {"scanpy": None}):
-            with self.assertRaises(NotImplementedError):
-                self.data.compute_embedding("umap", filter)
-
-        # if we happen to have scanpy, test the full API, else punt
-        import importlib
-
-        scanpy_spec = importlib.util.find_spec("scanpy")
-        if scanpy_spec is None:
-            print("Skipping compute_embedding test as ScanPy not installed")
-            return
-
-        # this feature is unsupported in backed mode, and we expect an error
-        if self.data.data.isbacked:
-            with self.assertRaises(NotImplementedError):
-                self.data.compute_embedding("umap", filter)
-            return
-
-        schema = self.data.compute_embedding("umap", filter)
-
-        self.assertIsInstance(schema["name"], str)
-        name = schema["name"]
-        self.assertEqual(schema["type"], "float32")
-        self.assertEqual(schema["dims"], [f"{name}_0", f"{name}_1"])
-
-        emb = self.data.data.obsm[f"X_{name}"]
-        self.assertEqual(emb.shape, (2638, 2))
-        self.assertTrue(np.isfinite(emb[0:100]).all())
-        self.assertTrue(np.isnan(emb[100:]).all())

client/__tests__/e2e/test_config.yaml

@@ -25,4 +25,3 @@ dataset:
 
   embeddings:
     names: []
-    enable_reembedding: false

client/src/actions/embedding.js

@@ -11,7 +11,7 @@ export async function _switchEmbedding(
   newEmbeddingName
 ) {
   /*
-  DRY helper used by this and reembedding action creators
+  DRY helper used by embedding action creators
   */
   const base = prevAnnoMatrix.base();
   const embeddingDf = await base.fetch("emb", newEmbeddingName);

client/src/actions/index.js

@@ -5,9 +5,6 @@ import {
   doJsonRequest,
   dispatchNetworkErrorMessageToUser,
 } from "../util/actionHelpers";
-import {
-  requestReembed /* , reembedResetWorldToUniverse -- disabled temporarily, TODO issue #1606 */,
-} from "./reembed";
 import { loadUserColorConfig } from "../util/stateManager/colorHelpers";
 import * as selnActions from "./selection";
 import * as annoActions from "./annotation";
@@ -243,7 +240,6 @@ export default {
   requestDifferentialExpression,
   requestSingleGeneExpressionCountsForColoringPOST,
   requestUserDefinedGene,
-  requestReembed,
   selectContinuousMetadataAction: selnActions.selectContinuousMetadataAction,
   selectCategoricalMetadataAction: selnActions.selectCategoricalMetadataAction,
   selectCategoricalAllMetadataAction:

client/src/actions/reembed.js

@@ -1,112 +0,0 @@
-import { API } from "../globals";
-import {
-  postNetworkErrorToast,
-  postAsyncSuccessToast,
-  postAsyncFailureToast,
-} from "../components/framework/toasters";
-import { _switchEmbedding } from "./embedding";
-
-function abortableFetch(request, opts, timeout = 0) {
-  const controller = new AbortController();
-  const { signal } = controller;
-
-  return {
-    abort: () => controller.abort(),
-    isAborted: () => signal.aborted,
-    ready: () => {
-      if (timeout) {
-        setTimeout(() => controller.abort(), timeout);
-      }
-      return fetch(request, { ...opts, signal });
-    },
-  };
-}
-
-async function doReembedFetch(dispatch, getState) {
-  const state = getState();
-  let cells = state.annoMatrix.rowIndex.labels();
-
-  // These lines ensure that we convert any TypedArray to an Array.
-  // This is necessary because JSON.stringify() does some very strange
-  // things with TypedArrays (they are marshalled to JSON objects, rather
-  // than being marshalled as a JSON array).
-  cells = Array.isArray(cells) ? cells : Array.from(cells);
-
-  const af = abortableFetch(
-    `${API.prefix}${API.version}layout/obs`,
-    {
-      method: "PUT",
-      headers: new Headers({
-        Accept: "application/octet-stream",
-        "Content-Type": "application/json",
-      }),
-      body: JSON.stringify({
-        method: "umap",
-        filter: { obs: { index: cells } },
-      }),
-      credentials: "include",
-    },
-    60000 // 1 minute timeout
-  );
-  dispatch({
-    type: "reembed: request start",
-    abortableFetch: af,
-  });
-  const res = await af.ready();
-
-  if (res.ok && res.headers.get("Content-Type").includes("application/json")) {
-    return res;
-  }
-
-  // else an error
-  let msg = `Unexpected HTTP response ${res.status}, ${res.statusText}`;
-  const body = await res.text();
-  if (body && body.length > 0) {
-    msg = `${msg} -- ${body}`;
-  }
-  throw new Error(msg);
-}
-
-/*
-functions below are dispatch-able
-*/
-export function requestReembed() {
-  return async (dispatch, getState) => {
-    try {
-      const res = await doReembedFetch(dispatch, getState);
-      const schema = await res.json();
-      dispatch({
-        type: "reembed: request completed",
-      });
-
-      const {
-        annoMatrix: prevAnnoMatrix,
-        obsCrossfilter: prevCrossfilter,
-      } = getState();
-      const base = prevAnnoMatrix.base().addEmbedding(schema);
-      const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
-        base,
-        prevCrossfilter,
-        schema.name
-      );
-      dispatch({
-        type: "reembed: add reembedding",
-        schema,
-        annoMatrix,
-        obsCrossfilter,
-      });
-
-      postAsyncSuccessToast("Re-embedding has completed.");
-    } catch (error) {
-      dispatch({
-        type: "reembed: request aborted",
-      });
-      if (error.name === "AbortError") {
-        postAsyncFailureToast("Re-embedding calculation was aborted.");
-      } else {
-        postNetworkErrorToast(`Re-embedding: ${error.message}`);
-      }
-      console.log("Reembed exception:", error, error.name, error.message);
-    }
-  };
-}

client/src/components/menubar/index.js

@@ -11,7 +11,6 @@ import AuthButtons from "./authButtons";
 import Subset from "./subset";
 import UndoRedoReset from "./undoRedo";
 import DiffexpButtons from "./diffexpButtons";
-import Reembedding from "./reembedding";
 import { getEmbSubsetView } from "../../util/stateManager/viewStackHelpers";
 
 @connect((state) => {
@@ -49,8 +48,6 @@ import { getEmbSubsetView } from "../../util/stateManager/viewStackHelpers";
     tosURL: state.config?.parameters?.about_legal_tos,
     privacyURL: state.config?.parameters?.about_legal_privacy,
     categoricalSelection: state.categoricalSelection,
-    enableReembedding:
-      state.config?.parameters?.["enable-reembedding"] ?? false,
   };
 })
 class MenuBar extends React.PureComponent {
@@ -212,7 +209,6 @@ class MenuBar extends React.PureComponent {
       colorAccessor,
       subsetPossible,
       subsetResetPossible,
-      enableReembedding,
       userInfo,
       auth,
     } = this.props;
@@ -262,7 +258,6 @@ class MenuBar extends React.PureComponent {
             this.handleClipPercentileMinValueChange
           }
         />
-        {enableReembedding ? <Reembedding /> : null}
         <Tooltip
           content="When a category is colored by, show labels on the graph"
           position="bottom"

client/src/components/menubar/reembedding.js

@@ -1,40 +0,0 @@
-import React from "react";
-import { connect } from "react-redux";
-import { AnchorButton, ButtonGroup, Tooltip } from "@blueprintjs/core";
-import * as globals from "../../globals";
-import actions from "../../actions";
-import styles from "./menubar.css";
-
-@connect((state) => ({
-  reembedController: state.reembedController,
-  annoMatrix: state.annoMatrix,
-}))
-class Reembedding extends React.PureComponent {
-  render() {
-    const { dispatch, annoMatrix, reembedController } = this.props;
-    const loading = !!reembedController?.pendingFetch;
-    const disabled = annoMatrix.nObs === annoMatrix.schema.dataframe.nObs;
-    const tipContent = disabled
-      ? "Subset cells first, then click to recompute UMAP embedding."
-      : "Click to recompute UMAP embedding on the current cell subset.";
-
-    return (
-      <ButtonGroup className={styles.menubarButton}>
-        <Tooltip
-          content={tipContent}
-          position="bottom"
-          hoverOpenDelay={globals.tooltipHoverOpenDelay}
-        >
-          <AnchorButton
-            icon="new-object"
-            disabled={disabled}
-            onClick={() => dispatch(actions.requestReembed())}
-            loading={loading}
-          />
-        </Tooltip>
-      </ButtonGroup>
-    );
-  }
-}
-
-export default Reembedding;

client/src/reducers/index.js

@@ -20,7 +20,6 @@ import genesetsUI from "./genesetsUI";
 import autosave from "./autosave";
 import centroidLabels from "./centroidLabels";
 import pointDialation from "./pointDilation";
-import { reembedController } from "./reembed";
 import { gcMiddleware as annoMatrixGC } from "../annoMatrix";
 
 import undoableConfig from "./undoableConfig";
@@ -42,7 +41,6 @@ const Reducer = undoable(
     ["differential", differential],
     ["centroidLabels", centroidLabels],
     ["pointDilation", pointDialation],
-    ["reembedController", reembedController],
     ["autosave", autosave],
     ["userInfo", userInfo],
   ]),

client/src/reducers/layoutChoice.js

@@ -47,19 +47,6 @@ const LayoutChoice = (
       return { ...state, current, currentDimNames };
     }
 
-    case "reembed: add reembedding": {
-      const { schema } = nextSharedState.annoMatrix;
-      const { name } = action.schema;
-      const available = Array.from(new Set(state.available).add(name));
-      const currentDimNames = schema.layout.obsByName[name].dims;
-      return {
-        ...state,
-        available,
-        current: name,
-        currentDimNames,
-      };
-    }
-
     default: {
       return state;
     }

client/src/reducers/reembed.js

@@ -1,29 +0,0 @@
-/*
-controller state is not part of the undo/redo history
-*/
-export const reembedController = (
-  state = {
-    pendingFetch: null,
-  },
-  action
-) => {
-  switch (action.type) {
-    case "reembed: request start": {
-      return {
-        ...state,
-        pendingFetch: action.abortableFetch,
-      };
-    }
-    case "reembed: request aborted":
-    case "reembed: request cancel":
-    case "reembed: request completed": {
-      return {
-        ...state,
-        pendingFetch: null,
-      };
-    }
-    default: {
-      return state;
-    }
-  }
-};