remove experimental reembedding support (#2301)

* remove experimental reembedding support

* lint

* lint

* add prepare requirements to requirements-dev

* oops, revert accidental deletion of import

* more test modifications

* remove obsolete unit tests

backend/czi_hosted/app/app.py

@@ -312,11 +312,6 @@ class LayoutObsAPI(DatasetResource):
     def get(self, data_adaptor):
         return common_rest.layout_obs_get(request, data_adaptor)
 
-    @cache_control(no_store=True)
-    @rest_get_data_adaptor
-    def put(self, data_adaptor):
-        return common_rest.layout_obs_put(request, data_adaptor)
-
 
 class GenesetsAPI(DatasetResource):
     @cache_control(public=True, max_age=ONE_WEEK)

backend/czi_hosted/cli/launch.py

@@ -90,14 +90,6 @@ def config_args(func):
         metavar="<text>",
         help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all.",
     )
-    @click.option(
-        "--experimental-enable-reembedding",
-        is_flag=True,
-        default=DEFAULT_CONFIG.default_dataset_config.embeddings__enable_reembedding,
-        show_default=False,
-        hidden=True,
-        help="Enable experimental on-demand re-embedding using UMAP. WARNING: may be very slow.",
-    )
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
         return func(*args, **kwargs)
@@ -314,7 +306,6 @@ def launch(
     annotations_dir,
     backed,
     disable_diffexp,
-    experimental_enable_reembedding,
     config_file,
     dump_default_config,
 ):
@@ -376,7 +367,6 @@ def launch(
             presentation__max_categories=max_category_items,
             presentation__custom_colors=not disable_custom_colors,
             embeddings__names=embedding,
-            embeddings__enable_reembedding=experimental_enable_reembedding,
             diffexp__enable=not disable_diffexp,
             diffexp__lfc_cutoff=diffexp_lfc_cutoff,
         )

backend/czi_hosted/common/config/client_config.py

@@ -40,7 +40,6 @@ def get_client_config(app_config, data_adaptor):
         "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
         "backed": server_config.adaptor__anndata_adaptor__backed,
         "disable-diffexp": not dataset_config.diffexp__enable,
-        "enable-reembedding": dataset_config.embeddings__enable_reembedding,
         "annotations": False,
         "annotations_file": None,
         "annotations_dir": None,

backend/czi_hosted/common/config/dataset_config.py

@@ -6,8 +6,6 @@ from backend.czi_hosted.common.annotations.hosted_tiledb import AnnotationsHoste
 from backend.czi_hosted.common.annotations.local_file_csv import AnnotationsLocalFile
 from backend.czi_hosted.common.config.base_config import BaseConfig
 from backend.common.errors import ConfigurationError
-from backend.czi_hosted.compute.scanpy import get_scanpy_module
-from backend.czi_hosted.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
 from backend.czi_hosted.db.db_utils import DbUtils
 
 
@@ -41,7 +39,6 @@ class DatasetConfig(BaseConfig):
             ]["hosted_file_directory"]
 
             self.embeddings__names = default_config["embeddings"]["names"]
-            self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"]
 
             self.diffexp__enable = default_config["diffexp"]["enable"]
             self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
@@ -186,24 +183,6 @@ class DatasetConfig(BaseConfig):
 
     def handle_embeddings(self):
         self.validate_correct_type_of_configuration_attribute("embeddings__names", list)
-        self.validate_correct_type_of_configuration_attribute("embeddings__enable_reembedding", bool)
-
-        server_config = self.app_config.server_config
-        if self.embeddings__enable_reembedding:
-            if server_config.single_dataset__datapath:
-                matrix_data_loader = MatrixDataLoader(
-                    server_config.single_dataset__datapath, app_config=self.app_config
-                )
-                if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD:
-                    raise ConfigurationError("enable-reembedding is only supported with H5AD files.")
-                if server_config.adaptor__anndata_adaptor__backed:
-                    raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
-
-            try:
-                get_scanpy_module()
-            except NotImplementedError:
-                # Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported
-                raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
 
     def handle_diffexp(self, context):
         self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool)

backend/czi_hosted/common/rest.py

@@ -312,25 +312,6 @@ def layout_obs_get(request, data_adaptor):
         )
 
 
-def layout_obs_put(request, data_adaptor):
-    if not data_adaptor.dataset_config.embeddings__enable_reembedding:
-        return abort(HTTPStatus.NOT_IMPLEMENTED)
-
-    args = request.get_json()
-    filter = args["filter"] if args else None
-    if not filter:
-        return abort_and_log(HTTPStatus.BAD_REQUEST, "obs filter is required")
-    method = args["method"] if args else "umap"
-
-    try:
-        schema = data_adaptor.compute_embedding(method, filter)
-        return make_response(jsonify(schema), HTTPStatus.OK, {"Content-Type": "application/json"})
-    except NotImplementedError as e:
-        return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, str(e))
-    except (ValueError, DisabledFeatureError, FilterError) as e:
-        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
-
-
 def genesets_get(request, data_adaptor):
     preferred_mimetype = request.accept_mimetypes.best_match(["application/json", "text/csv"])
     if preferred_mimetype not in ("application/json", "text/csv"):

backend/czi_hosted/compute/scanpy.py

@@ -1,53 +0,0 @@
-import importlib
-import numpy as np
-
-"""
-Wrapper for various scanpy modules.  Will raise NotImplementedError if the scanpy
-module is not installed/available
-"""
-
-
-def get_scanpy_module():
-    try:
-        sc = importlib.import_module("scanpy")
-        # Future: we could enforce versions here, eg, lookat sc.__version__
-        return sc
-    except ModuleNotFoundError as e:
-        raise NotImplementedError("Please install scanpy to enable UMAP re-embedding") from e
-    except Exception as e:
-        # will capture other ImportError corner cases
-        raise NotImplementedError() from e
-
-
-def scanpy_umap(adata, obs_mask=None, pca_options={}, neighbors_options={}, umap_options={}):
-    """
-    Given adata and an obs mask, return a new embedding for adata[obs_mask, :]
-    as an ndarray of shape (len(obs_mask), N), where N>=2.
-
-    Do NOT mutate adata.
-    """
-
-    # backed mode is incompatible with the current implementation
-    if adata.isbacked:
-        raise NotImplementedError("Backed mode is incompatible with re-embedding")
-
-    # safely get scanpy module, which may not be present.
-    sc = get_scanpy_module()
-
-    # https://github.com/theislab/anndata/issues/311
-    obs_mask = slice(None) if obs_mask is None else obs_mask
-    adata = adata[obs_mask, :].copy()
-
-    for k in list(adata.obsm.keys()):
-        del adata.obsm[k]
-    for k in list(adata.uns.keys()):
-        del adata.uns[k]
-
-    sc.pp.pca(adata, zero_center=None, n_comps=min(adata.n_vars - 1, 50), **pca_options)
-    sc.pp.neighbors(adata, **neighbors_options)
-    sc.tl.umap(adata, **umap_options)
-
-    umap = adata.obsm["X_umap"]
-    result = np.full((obs_mask.shape[0], umap.shape[1]), np.NaN)
-    result[obs_mask] = umap
-    return result

backend/czi_hosted/data_anndata/anndata_adaptor.py

@@ -1,20 +1,17 @@
 import warnings
-from datetime import datetime
 
 import anndata
 import numpy as np
 from packaging import version
 from pandas.core.dtypes.dtypes import CategoricalDtype
 from scipy import sparse
-from server_timing import Timing as ServerTiming
 
 import backend.common.compute.diffexp_generic as diffexp_generic
 from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
 from backend.common.constants import Axis, MAX_LAYOUTS
 from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata
-from backend.common.errors import PrepareError, DatasetAccessError, FilterError
+from backend.common.errors import PrepareError, DatasetAccessError
 from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
-from backend.czi_hosted.compute.scanpy import scanpy_umap
 from backend.czi_hosted.data_common.data_adaptor import DataAdaptor
 from backend.common.fbs.matrix import encode_matrix_fbs
 
@@ -301,28 +298,6 @@ class AnndataAdaptor(DataAdaptor):
         full_embedding = self.data.obsm[f"X_{ename}"]
         return full_embedding[:, 0:dims]
 
-    def compute_embedding(self, method, obsFilter):
-        if Axis.VAR in obsFilter:
-            raise FilterError("Observation filters may not contain variable conditions")
-        if method != "umap":
-            raise NotImplementedError(f"re-embedding method {method} is not available.")
-        try:
-            shape = self.get_shape()
-            obs_mask = self._axis_filter_to_mask(Axis.OBS, obsFilter["obs"], shape[0])
-        except (KeyError, IndexError):
-            raise FilterError("Error parsing filter")
-        with ServerTiming.time("layout.compute"):
-            X_umap = scanpy_umap(self.data, obs_mask)
-
-        # Server picks reemedding name, which must not collide with any other
-        # embedding name generated by this backend.
-        name = f"reembed:{method}_{datetime.now().isoformat(timespec='milliseconds')}"
-        dims = [f"{name}_0", f"{name}_1"]
-        layout_schema = {"name": name, "type": "float32", "dims": dims}
-        self.schema["layout"]["obs"].append(layout_schema)
-        self.data.obsm[f"X_{name}"] = X_umap
-        return layout_schema
-
     def compute_diffexp_ttest(self, maskA, maskB, top_n=None, lfc_cutoff=None):
         if top_n is None:
             top_n = self.dataset_config.diffexp__top_n

backend/czi_hosted/data_common/data_adaptor.py

@@ -71,11 +71,6 @@ class DataAdaptor(metaclass=ABCMeta):
         """return an numpy array for the given pre-computed embedding name."""
         pass
 
-    @abstractmethod
-    def compute_embedding(self, method, filter):
-        """compute a new embedding on the specified obs subset, and return the embedding schema. """
-        pass
-
     @abstractmethod
     def get_X_array(self, obs_mask=None, var_mask=None):
         """return the X array, possibly filtered by obs_mask or var_mask.

backend/czi_hosted/data_cxg/cxg_adaptor.py

@@ -199,9 +199,6 @@ class CxgAdaptor(DataAdaptor):
         array = self.open_array(f"emb/{ename}")
         return array[:, 0:dims]
 
-    def compute_embedding(self, method, filter):
-        raise NotImplementedError("CXG does not yet support re-embedding")
-
     def compute_diffexp_ttest(self, maskA, maskB, top_n=None, lfc_cutoff=None):
         if top_n is None:
             top_n = self.dataset_config.diffexp__top_n

backend/czi_hosted/default_config.py

@@ -197,7 +197,6 @@ dataset:
 
   embeddings:
     names : []
-    enable_reembedding: false
 
   diffexp:
     enable: true

backend/czi_hosted/eb/app.py

@@ -164,7 +164,6 @@ try:
         app_config.update_server_config(multi_dataset__dataroot=dataroot)
 
     # overwrite configuration for the eb app
-    app_config.update_default_dataset_config(embeddings__enable_reembedding=False,)
     app_config.update_server_config(multi_dataset__allowed_matrix_types=["cxg"],)
 
     # complete config

backend/czi_hosted/requirements-dev.txt

@@ -8,4 +8,5 @@ pytest>=3.6.3
 python-jose>=3.2.0
 twine>=1.12.1
 -r requirements.txt
+-r requirements-prepare.txt
 rsa>=4.7 # not directly required, pinned by Snyk to avoid a vulnerability

backend/czi_hosted/requirements-prepare.txt

@@ -1,2 +1,4 @@
 python-igraph
 louvain>=0.6
+scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here
+umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn

backend/czi_hosted/requirements.txt

@@ -21,6 +21,4 @@ scipy>=1.0
 requests>=2.22.0
 tiledb>=0.5.9,>=0.6.2,!=0.7.2, !=0.8.6
 s3fs==0.4.2
-scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here
 sqlalchemy>=1.3.18
-umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn

backend/server/app/app.py

@@ -187,11 +187,6 @@ class LayoutObsAPI(Resource):
     def get(self, data_adaptor):
         return common_rest.layout_obs_get(request, data_adaptor)
 
-    @cache_control(no_store=True)
-    @rest_get_data_adaptor
-    def put(self, data_adaptor):
-        return common_rest.layout_obs_put(request, data_adaptor)
-
 
 class GenesetsAPI(Resource):
     @cache_control(public=True, max_age=ONE_WEEK)

backend/server/cli/launch.py

@@ -107,14 +107,6 @@ def config_args(func):
         metavar="<text>",
         help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all.",
     )
-    @click.option(
-        "--experimental-enable-reembedding",
-        is_flag=True,
-        default=DEFAULT_CONFIG.dataset_config.embeddings__enable_reembedding,
-        show_default=False,
-        hidden=True,
-        help="Enable experimental on-demand re-embedding using UMAP. WARNING: may be very slow.",
-    )
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
         return func(*args, **kwargs)
@@ -324,7 +316,6 @@ def launch(
     disable_gene_sets_save,
     backed,
     disable_diffexp,
-    experimental_enable_reembedding,
     config_file,
     dump_default_config,
 ):
@@ -383,7 +374,6 @@ def launch(
             presentation__max_categories=max_category_items,
             presentation__custom_colors=not disable_custom_colors,
             embeddings__names=embedding,
-            embeddings__enable_reembedding=experimental_enable_reembedding,
             diffexp__enable=not disable_diffexp,
             diffexp__lfc_cutoff=diffexp_lfc_cutoff,
         )

backend/server/common/config/client_config.py

@@ -40,7 +40,6 @@ def get_client_config(app_config, data_adaptor):
         "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
         "backed": server_config.adaptor__anndata_adaptor__backed,
         "disable-diffexp": not dataset_config.diffexp__enable,
-        "enable-reembedding": dataset_config.embeddings__enable_reembedding,
         "annotations": False,
         "annotations_file": None,
         "annotations_dir": None,

backend/server/common/config/dataset_config.py

@@ -4,7 +4,6 @@ from os.path import splitext, isdir
 from backend.server.common.annotations.local_file_csv import AnnotationsLocalFile
 from backend.server.common.config.base_config import BaseConfig
 from backend.common.errors import ConfigurationError, AnnotationsError
-from backend.server.compute.scanpy import get_scanpy_module
 from backend.server.data_common.matrix_loader import MatrixDataLoader
 
 
@@ -34,7 +33,6 @@ class DatasetConfig(BaseConfig):
             ]["gene_sets_file"]
 
             self.embeddings__names = default_config["embeddings"]["names"]
-            self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"]
 
             self.diffexp__enable = default_config["diffexp"]["enable"]
             self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
@@ -173,19 +171,6 @@ class DatasetConfig(BaseConfig):
 
     def handle_embeddings(self):
         self.validate_correct_type_of_configuration_attribute("embeddings__names", list)
-        self.validate_correct_type_of_configuration_attribute("embeddings__enable_reembedding", bool)
-
-        server_config = self.app_config.server_config
-        if self.embeddings__enable_reembedding:
-            if server_config.single_dataset__datapath:
-                if server_config.adaptor__anndata_adaptor__backed:
-                    raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
-
-            try:
-                get_scanpy_module()
-            except NotImplementedError:
-                # Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported
-                raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
 
     def handle_diffexp(self, context):
         self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool)

backend/server/common/rest.py

@@ -311,25 +311,6 @@ def layout_obs_get(request, data_adaptor):
         )
 
 
-def layout_obs_put(request, data_adaptor):
-    if not data_adaptor.dataset_config.embeddings__enable_reembedding:
-        return abort(HTTPStatus.NOT_IMPLEMENTED)
-
-    args = request.get_json()
-    filter = args["filter"] if args else None
-    if not filter:
-        return abort_and_log(HTTPStatus.BAD_REQUEST, "obs filter is required")
-    method = args["method"] if args else "umap"
-
-    try:
-        schema = data_adaptor.compute_embedding(method, filter)
-        return make_response(jsonify(schema), HTTPStatus.OK, {"Content-Type": "application/json"})
-    except NotImplementedError as e:
-        return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, str(e))
-    except (ValueError, DisabledFeatureError, FilterError) as e:
-        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
-
-
 def genesets_get(request, data_adaptor):
     preferred_mimetype = request.accept_mimetypes.best_match(["application/json", "text/csv"])
     if preferred_mimetype not in ("application/json", "text/csv"):

backend/server/compute/scanpy.py

@@ -1,53 +0,0 @@
-import importlib
-import numpy as np
-
-"""
-Wrapper for various scanpy modules.  Will raise NotImplementedError if the scanpy
-module is not installed/available
-"""
-
-
-def get_scanpy_module():
-    try:
-        sc = importlib.import_module("scanpy")
-        # Future: we could enforce versions here, eg, lookat sc.__version__
-        return sc
-    except ModuleNotFoundError as e:
-        raise NotImplementedError("Please install scanpy to enable UMAP re-embedding") from e
-    except Exception as e:
-        # will capture other ImportError corner cases
-        raise NotImplementedError() from e
-
-
-def scanpy_umap(adata, obs_mask=None, pca_options={}, neighbors_options={}, umap_options={}):
-    """
-    Given adata and an obs mask, return a new embedding for adata[obs_mask, :]
-    as an ndarray of shape (len(obs_mask), N), where N>=2.
-
-    Do NOT mutate adata.
-    """
-
-    # backed mode is incompatible with the current implementation
-    if adata.isbacked:
-        raise NotImplementedError("Backed mode is incompatible with re-embedding")
-
-    # safely get scanpy module, which may not be present.
-    sc = get_scanpy_module()
-
-    # https://github.com/theislab/anndata/issues/311
-    obs_mask = slice(None) if obs_mask is None else obs_mask
-    adata = adata[obs_mask, :].copy()
-
-    for k in list(adata.obsm.keys()):
-        del adata.obsm[k]
-    for k in list(adata.uns.keys()):
-        del adata.uns[k]
-
-    sc.pp.pca(adata, zero_center=None, n_comps=min(adata.n_vars - 1, 50), **pca_options)
-    sc.pp.neighbors(adata, **neighbors_options)
-    sc.tl.umap(adata, **umap_options)
-
-    umap = adata.obsm["X_umap"]
-    result = np.full((obs_mask.shape[0], umap.shape[1]), np.NaN)
-    result[obs_mask] = umap
-    return result

backend/server/data_anndata/anndata_adaptor.py

@@ -1,20 +1,17 @@
 import warnings
-from datetime import datetime
 
 import anndata
 import numpy as np
 from packaging import version
 from pandas.core.dtypes.dtypes import CategoricalDtype
 from scipy import sparse
-from server_timing import Timing as ServerTiming
 
 import backend.common.compute.diffexp_generic as diffexp_generic
 from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
 from backend.common.constants import Axis, MAX_LAYOUTS
 from backend.server.common.corpora import corpora_get_props_from_anndata
-from backend.common.errors import PrepareError, DatasetAccessError, FilterError
+from backend.common.errors import PrepareError, DatasetAccessError
 from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
-from backend.server.compute.scanpy import scanpy_umap
 from backend.server.data_common.data_adaptor import DataAdaptor
 from backend.common.fbs.matrix import encode_matrix_fbs
 
@@ -301,28 +298,6 @@ class AnndataAdaptor(DataAdaptor):
         full_embedding = self.data.obsm[f"X_{ename}"]
         return full_embedding[:, 0:dims]
 
-    def compute_embedding(self, method, obsFilter):
-        if Axis.VAR in obsFilter:
-            raise FilterError("Observation filters may not contain variable conditions")
-        if method != "umap":
-            raise NotImplementedError(f"re-embedding method {method} is not available.")
-        try:
-            shape = self.get_shape()
-            obs_mask = self._axis_filter_to_mask(Axis.OBS, obsFilter["obs"], shape[0])
-        except (KeyError, IndexError):
-            raise FilterError("Error parsing filter")
-        with ServerTiming.time("layout.compute"):
-            X_umap = scanpy_umap(self.data, obs_mask)
-
-        # Server picks reemedding name, which must not collide with any other
-        # embedding name generated by this backend.
-        name = f"reembed:{method}_{datetime.now().isoformat(timespec='milliseconds')}"
-        dims = [f"{name}_0", f"{name}_1"]
-        layout_schema = {"name": name, "type": "float32", "dims": dims}
-        self.schema["layout"]["obs"].append(layout_schema)
-        self.data.obsm[f"X_{name}"] = X_umap
-        return layout_schema
-
     def compute_diffexp_ttest(self, maskA, maskB, top_n=None, lfc_cutoff=None):
         if top_n is None:
             top_n = self.dataset_config.diffexp__top_n

backend/server/data_common/data_adaptor.py

@@ -66,11 +66,6 @@ class DataAdaptor(metaclass=ABCMeta):
         """return an numpy array for the given pre-computed embedding name."""
         pass
 
-    @abstractmethod
-    def compute_embedding(self, method, filter):
-        """compute a new embedding on the specified obs subset, and return the embedding schema."""
-        pass
-
     @abstractmethod
     def get_X_array(self, obs_mask=None, var_mask=None):
         """return the X array, possibly filtered by obs_mask or var_mask.

backend/server/default_config.py

@@ -71,7 +71,6 @@ dataset:
 
   embeddings:
     names : []
-    enable_reembedding: false
 
   diffexp:
     enable: true

backend/server/requirements-dev.txt

@@ -8,3 +8,4 @@ pytest>=3.6.3
 python-jose>=3.2.0
 twine>=1.12.1
 -r requirements.txt
+-r requirements-prepare.txt

backend/server/requirements-prepare.txt

@@ -1,2 +1,4 @@
 python-igraph>=0.8
 louvain>=0.6
+scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here
+umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn

backend/server/requirements.txt

@@ -21,5 +21,3 @@ PyYAML>=5.4  # CVE-2020-14343
 scipy>=1.4
 requests>=2.22.0
 s3fs==0.4.2
-scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here
-umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn

backend/test/fixtures/czi_hosted_dataset_config_outline.py

@@ -25,7 +25,6 @@ dataset:
 
   embeddings:
     names: {embedding_names}
-    enable_reembedding: {enable_reembedding}
 
   diffexp:
     enable: {enable_difexp}

backend/test/fixtures/dataset_config_outline.py

@@ -22,7 +22,6 @@ dataset:
 
   embeddings:
     names: {embedding_names}
-    enable_reembedding: {enable_reembedding}
 
   diffexp:
     enable: {enable_difexp}

backend/test/test_czi_hosted/unit/__init__.py

@@ -170,7 +170,6 @@ class BaseTest(unittest.TestCase):
                 multi_dataset__index=True,
                 multi_dataset__allowed_matrix_types=["cxg"]
             )
-            app_config.update_default_dataset_config(embeddings__enable_reembedding=False, )
         app_config.complete_config(logging.info)
 
         app = TestServer(app_config).app

backend/test/test_czi_hosted/unit/common/config/__init__.py

@@ -125,7 +125,6 @@ class ConfigTests(BaseTest):
         local_file_csv_directory="null",
         local_file_csv_file="null",
         embedding_names=[],
-        enable_reembedding="false",
         enable_difexp="true",
         lfc_cutoff=0.01,
         top_n=10,
@@ -192,7 +191,6 @@ class ConfigTests(BaseTest):
             local_file_csv_directory=local_file_csv_directory,
             local_file_csv_file=local_file_csv_file,
             embedding_names=embedding_names,
-            enable_reembedding=enable_reembedding,
             enable_difexp=enable_difexp,
             lfc_cutoff=lfc_cutoff,
             top_n=top_n,
@@ -228,7 +226,6 @@ class ConfigTests(BaseTest):
         local_file_csv_directory="null",
         local_file_csv_file="null",
         embedding_names=[],
-        enable_reembedding="false",
         enable_difexp="true",
         lfc_cutoff=0.01,
         top_n=10,

backend/test/test_czi_hosted/unit/common/config/test_dataset_config.py

@@ -49,7 +49,7 @@ class TestDatasetConfig(ConfigTests):
     def test_complete_config_checks_all_attr(self, mock_check_attrs):
         mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
         self.dataset_config.complete_config(self.context)
-        self.assertEqual(mock_check_attrs.call_count, 19)
+        self.assertEqual(mock_check_attrs.call_count, 18)
 
     def test_app_sets_script_vars(self):
         config = self.get_config(scripts=["path/to/script"])
@@ -130,20 +130,6 @@ class TestDatasetConfig(ConfigTests):
         cwd = os.getcwd()
         self.assertEqual(config.default_dataset_config.user_annotations._get_output_dir(), cwd)
 
-    def test_handle_embeddings__checks_data_file_types(self):
-        file_name = self.custom_app_config(
-            embedding_names=["name1", "name2"],
-            enable_reembedding="true",
-            dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
-            anndata_backed="true",
-            config_file_name=self.config_file_name,
-        )
-        config = AppConfig()
-        config.update_from_config_file(file_name)
-        config.server_config.complete_config(self.context)
-        with self.assertRaises(ConfigurationError):
-            config.default_dataset_config.handle_embeddings()
-
     def test_handle_diffexp__raises_warning_for_large_datasets(self):
         config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15)
         config.server_config.complete_config(self.context)

backend/test/test_czi_hosted/unit/common/test_api.py

@@ -69,35 +69,6 @@ class EndPoints(BaseTest):
         self.assertIsNone(df["row_idx"])
         self.assertEqual(len(df["columns"]), df["n_cols"])
 
-    def test_put_layout_fbs(self):
-        # first check that re-embedding is turned on
-        self.app.auth.get_user_id = lambda : "123"
-        result = self.client.get(f"{self.TEST_URL_BASE}config")
-        config_data = json.loads(result.data)
-        re_embed = config_data["config"]["parameters"]["enable-reembedding"]
-        if not re_embed:
-            return
-        # attempt to reembed with umap over 100 cells.
-        endpoint = "layout/obs"
-        url = f"{self.TEST_URL_BASE}{endpoint}"
-        data = {}
-        data["filter"] = {}
-        data["filter"]["obs"] = {}
-        data["filter"]["obs"]["index"] = list(range(100))
-        data["method"] = "umap"
-        result = self.client.put(url, json=data)
-
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        result_data = result.json()
-        self.assertIsInstance(result_data, dict)
-        self.assertEqual(result_data["type"], "float32")
-        self.assertTrue(result_data["name"].startswith("reembed:umap_"))
-        self.assertIsInstance(result_data["dims"], list)
-        self.assertEqual(len(result_data["dims"]), 2)
-        dims = result_data["dims"]
-        self.assertTrue(dims[0].startswith("reembed:umap_") and dims[0].endswith("_0"))
-        self.assertTrue(dims[1].startswith("reembed:umap_") and dims[1].endswith("_1"))
-
     def test_bad_filter(self):
         endpoint = "data/var"
         url = f"{self.TEST_URL_BASE}{endpoint}"
@@ -421,7 +392,7 @@ class EndPointsCxg(EndPoints):
     @classmethod
     def setUpClass(cls):
         app_config = AppConfig()
-        app_config.update_default_dataset_config(embeddings__enable_reembedding=True, user_annotations__enable=False)
+        app_config.update_default_dataset_config(user_annotations__enable=False)
 
     def test_get_genesets_json(self):
         self.app.auth.is_user_authenticated = lambda: True

backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor.py

@@ -196,39 +196,3 @@ class AdaptorTest(unittest.TestCase):
         self.assertEqual(data["n_rows"], 2638)
         self.assertEqual(data["n_cols"], 3)
         self.assertTrue((data["col_idx"] == [15, 1818, 1837]).all())
-
-    def test_compute_embedding(self):
-        filter = {"obs": {"index": [[0, 100]]}}
-
-        # Verify that we correctly handle the case where we lack scanpy
-        import unittest.mock
-
-        with unittest.mock.patch.dict(sys.modules, {"scanpy": None}):
-            with self.assertRaises(NotImplementedError):
-                self.data.compute_embedding("umap", filter)
-
-        # if we happen to have scanpy, test the full API, else punt
-        import importlib
-
-        scanpy_spec = importlib.util.find_spec("scanpy")
-        if scanpy_spec is None:
-            print("Skipping compute_embedding test as ScanPy not installed")
-            return
-
-        # this feature is unsupported in backed mode, and we expect an error
-        if self.data.data.isbacked:
-            with self.assertRaises(NotImplementedError):
-                self.data.compute_embedding("umap", filter)
-            return
-
-        schema = self.data.compute_embedding("umap", filter)
-
-        self.assertIsInstance(schema["name"], str)
-        name = schema["name"]
-        self.assertEqual(schema["type"], "float32")
-        self.assertEqual(schema["dims"], [f"{name}_0", f"{name}_1"])
-
-        emb = self.data.data.obsm[f"X_{name}"]
-        self.assertEqual(emb.shape, (2638, 2))
-        self.assertTrue(np.isfinite(emb[0:100]).all())
-        self.assertTrue(np.isnan(emb[100:]).all())

backend/test/test_server/unit/common/config/__init__.py

@@ -97,7 +97,6 @@ class ConfigTests(unittest.TestCase):
         local_file_csv_gene_sets_file="null",
         gene_sets_readonly="false",
         embedding_names=[],
-        enable_reembedding="false",
         enable_difexp="true",
         lfc_cutoff=0.01,
         top_n=10,
@@ -148,7 +147,6 @@ class ConfigTests(unittest.TestCase):
             local_file_csv_gene_sets_file=local_file_csv_gene_sets_file,
             gene_sets_readonly=gene_sets_readonly,
             embedding_names=embedding_names,
-            enable_reembedding=enable_reembedding,
             enable_difexp=enable_difexp,
             lfc_cutoff=lfc_cutoff,
             top_n=top_n,
@@ -184,7 +182,6 @@ class ConfigTests(unittest.TestCase):
         local_file_csv_gene_sets_file="null",
         gene_sets_readonly="false",
         embedding_names=[],
-        enable_reembedding="false",
         enable_difexp="true",
         lfc_cutoff=0.01,
         top_n=10,

backend/test/test_server/unit/common/config/test_dataset_config.py

@@ -46,7 +46,7 @@ class TestDatasetConfig(ConfigTests):
         mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
         self.dataset_config.complete_config(self.context)
         self.assertIsNotNone(self.config.server_config.data_adaptor)
-        self.assertEqual(mock_check_attrs.call_count, 17)
+        self.assertEqual(mock_check_attrs.call_count, 16)
 
     def test_app_sets_script_vars(self):
         config = self.get_config(scripts=["path/to/script"])
@@ -106,20 +106,6 @@ class TestDatasetConfig(ConfigTests):
         cwd = os.getcwd()
         self.assertEqual(config.dataset_config.user_annotations._get_output_dir(), cwd)
 
-    def test_handle_embeddings__checks_data_file_types(self):
-        file_name = self.custom_app_config(
-            embedding_names=["name1", "name2"],
-            enable_reembedding="true",
-            dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
-            anndata_backed="true",
-            config_file_name=self.config_file_name,
-        )
-        config = AppConfig()
-        config.update_from_config_file(file_name)
-        config.server_config.complete_config(self.context)
-        with self.assertRaises(ConfigurationError):
-            config.dataset_config.handle_embeddings()
-
     def test_handle_diffexp__raises_warning_for_large_datasets(self):
         config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15)
         config.server_config.complete_config(self.context)

backend/test/test_server/unit/common/test_api.py

@@ -73,34 +73,6 @@ class EndPoints(object):
         self.assertIsNone(df["row_idx"])
         self.assertEqual(len(df["columns"]), df["n_cols"])
 
-    def test_put_layout_fbs(self):
-        # first check that re-embedding is turned on
-        result = self.session.get(f"{self.URL_BASE}config")
-        config_data = result.json()
-        re_embed = config_data["config"]["parameters"]["enable-reembedding"]
-        if not re_embed:
-            return
-        # attempt to reembed with umap over 100 cells.
-        endpoint = "layout/obs"
-        url = f"{self.URL_BASE}{endpoint}"
-        data = {}
-        data["filter"] = {}
-        data["filter"]["obs"] = {}
-        data["filter"]["obs"]["index"] = list(range(100))
-        data["method"] = "umap"
-        result = self.session.put(url, json=data)
-
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        result_data = result.json()
-        self.assertIsInstance(result_data, dict)
-        self.assertEqual(result_data["type"], "float32")
-        self.assertTrue(result_data["name"].startswith("reembed:umap_"))
-        self.assertIsInstance(result_data["dims"], list)
-        self.assertEqual(len(result_data["dims"]), 2)
-        dims = result_data["dims"]
-        self.assertTrue(dims[0].startswith("reembed:umap_") and dims[0].endswith("_0"))
-        self.assertTrue(dims[1].startswith("reembed:umap_") and dims[1].endswith("_1"))
-
     def test_bad_filter(self):
         endpoint = "data/var"
         url = f"{self.URL_BASE}{endpoint}"
@@ -389,7 +361,6 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
                 f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
                 "--disable-annotations",
                 "--disable-gene-sets-save",
-                "--experimental-enable-reembedding",
             ],
         )
 

backend/test/test_server/unit/data_anndata/test_anndata_adaptor.py

@@ -198,39 +198,3 @@ class AdaptorTest(unittest.TestCase):
         self.assertEqual(data["n_rows"], 2638)
         self.assertEqual(data["n_cols"], 3)
         self.assertTrue((data["col_idx"] == [15, 1818, 1837]).all())
-
-    def test_compute_embedding(self):
-        filter = {"obs": {"index": [[0, 100]]}}
-
-        # Verify that we correctly handle the case where we lack scanpy
-        import unittest.mock
-
-        with unittest.mock.patch.dict(sys.modules, {"scanpy": None}):
-            with self.assertRaises(NotImplementedError):
-                self.data.compute_embedding("umap", filter)
-
-        # if we happen to have scanpy, test the full API, else punt
-        import importlib
-
-        scanpy_spec = importlib.util.find_spec("scanpy")
-        if scanpy_spec is None:
-            print("Skipping compute_embedding test as ScanPy not installed")
-            return
-
-        # this feature is unsupported in backed mode, and we expect an error
-        if self.data.data.isbacked:
-            with self.assertRaises(NotImplementedError):
-                self.data.compute_embedding("umap", filter)
-            return
-
-        schema = self.data.compute_embedding("umap", filter)
-
-        self.assertIsInstance(schema["name"], str)
-        name = schema["name"]
-        self.assertEqual(schema["type"], "float32")
-        self.assertEqual(schema["dims"], [f"{name}_0", f"{name}_1"])
-
-        emb = self.data.data.obsm[f"X_{name}"]
-        self.assertEqual(emb.shape, (2638, 2))
-        self.assertTrue(np.isfinite(emb[0:100]).all())
-        self.assertTrue(np.isnan(emb[100:]).all())