fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-25 04:08:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

remove experimental reembedding support (#2301)

Browse Source 
* remove experimental reembedding support

* lint

* lint

* add prepare requirements to requirements-dev

* oops, revert accidental deletion of import

* more test modifications

* remove obsolete unit tests
This commit is contained in:
Bruce Martin
2021-07-15 13:55:26 -07:00
committed by GitHub
parent e334fbe96e
commit 0667ad0274
46 changed files with 12 additions and 661 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/czi_hosted/app/app.py
-5
View File
@@ -312,11 +312,6 @@ class LayoutObsAPI(DatasetResource):
def get(self, data_adaptor):
return common_rest.layout_obs_get(request, data_adaptor)
@cache_control(no_store=True)
@rest_get_data_adaptor
def put(self, data_adaptor):
return common_rest.layout_obs_put(request, data_adaptor)
class GenesetsAPI(DatasetResource):
@cache_control(public=True, max_age=ONE_WEEK)
backend/czi_hosted/cli/launch.py
-10
View File
@@ -90,14 +90,6 @@ def config_args(func):
metavar="<text>",
help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all.",
)
@click.option(
"--experimental-enable-reembedding",
is_flag=True,
default=DEFAULT_CONFIG.default_dataset_config.embeddings__enable_reembedding,
show_default=False,
hidden=True,
help="Enable experimental on-demand re-embedding using UMAP. WARNING: may be very slow.",
)
@functools.wraps(func)
def wrapper(*args, **kwargs):
return func(*args, **kwargs)
@@ -314,7 +306,6 @@ def launch(
annotations_dir,
backed,
disable_diffexp,
experimental_enable_reembedding,
config_file,
dump_default_config,
):
@@ -376,7 +367,6 @@ def launch(
presentation__max_categories=max_category_items,
presentation__custom_colors=not disable_custom_colors,
embeddings__names=embedding,
embeddings__enable_reembedding=experimental_enable_reembedding,
diffexp__enable=not disable_diffexp,
diffexp__lfc_cutoff=diffexp_lfc_cutoff,
)
backend/czi_hosted/common/config/client_config.py
-1
View File
@@ -40,7 +40,6 @@ def get_client_config(app_config, data_adaptor):
"diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
"backed": server_config.adaptor__anndata_adaptor__backed,
"disable-diffexp": not dataset_config.diffexp__enable,
"enable-reembedding": dataset_config.embeddings__enable_reembedding,
"annotations": False,
"annotations_file": None,
"annotations_dir": None,
backend/czi_hosted/common/config/dataset_config.py
-21
View File
@@ -6,8 +6,6 @@ from backend.czi_hosted.common.annotations.hosted_tiledb import AnnotationsHoste
from backend.czi_hosted.common.annotations.local_file_csv import AnnotationsLocalFile
from backend.czi_hosted.common.config.base_config import BaseConfig
from backend.common.errors import ConfigurationError
from backend.czi_hosted.compute.scanpy import get_scanpy_module
from backend.czi_hosted.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
from backend.czi_hosted.db.db_utils import DbUtils
@@ -41,7 +39,6 @@ class DatasetConfig(BaseConfig):
]["hosted_file_directory"]
self.embeddings__names = default_config["embeddings"]["names"]
self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"]
self.diffexp__enable = default_config["diffexp"]["enable"]
self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
@@ -186,24 +183,6 @@ class DatasetConfig(BaseConfig):
def handle_embeddings(self):
self.validate_correct_type_of_configuration_attribute("embeddings__names", list)
self.validate_correct_type_of_configuration_attribute("embeddings__enable_reembedding", bool)
server_config = self.app_config.server_config
if self.embeddings__enable_reembedding:
if server_config.single_dataset__datapath:
matrix_data_loader = MatrixDataLoader(
server_config.single_dataset__datapath, app_config=self.app_config
)
if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD:
raise ConfigurationError("enable-reembedding is only supported with H5AD files.")
if server_config.adaptor__anndata_adaptor__backed:
raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
try:
get_scanpy_module()
except NotImplementedError:
# Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported
raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
def handle_diffexp(self, context):
self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool)
backend/czi_hosted/common/rest.py
-19
View File
@@ -312,25 +312,6 @@ def layout_obs_get(request, data_adaptor):
)
def layout_obs_put(request, data_adaptor):
if not data_adaptor.dataset_config.embeddings__enable_reembedding:
return abort(HTTPStatus.NOT_IMPLEMENTED)
args = request.get_json()
filter = args["filter"] if args else None
if not filter:
return abort_and_log(HTTPStatus.BAD_REQUEST, "obs filter is required")
method = args["method"] if args else "umap"
try:
schema = data_adaptor.compute_embedding(method, filter)
return make_response(jsonify(schema), HTTPStatus.OK, {"Content-Type": "application/json"})
except NotImplementedError as e:
return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, str(e))
except (ValueError, DisabledFeatureError, FilterError) as e:
return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
def genesets_get(request, data_adaptor):
preferred_mimetype = request.accept_mimetypes.best_match(["application/json", "text/csv"])
if preferred_mimetype not in ("application/json", "text/csv"):
backend/czi_hosted/compute/scanpy.py
-53
View File
@@ -1,53 +0,0 @@
import importlib
import numpy as np
"""
Wrapper for various scanpy modules. Will raise NotImplementedError if the scanpy
module is not installed/available
"""
def get_scanpy_module():
try:
sc = importlib.import_module("scanpy")
# Future: we could enforce versions here, eg, lookat sc.__version__
return sc
except ModuleNotFoundError as e:
raise NotImplementedError("Please install scanpy to enable UMAP re-embedding") from e
except Exception as e:
# will capture other ImportError corner cases
raise NotImplementedError() from e
def scanpy_umap(adata, obs_mask=None, pca_options={}, neighbors_options={}, umap_options={}):
"""
Given adata and an obs mask, return a new embedding for adata[obs_mask, :]
as an ndarray of shape (len(obs_mask), N), where N>=2.
Do NOT mutate adata.
"""
# backed mode is incompatible with the current implementation
if adata.isbacked:
raise NotImplementedError("Backed mode is incompatible with re-embedding")
# safely get scanpy module, which may not be present.
sc = get_scanpy_module()
# https://github.com/theislab/anndata/issues/311
obs_mask = slice(None) if obs_mask is None else obs_mask
adata = adata[obs_mask, :].copy()
for k in list(adata.obsm.keys()):
del adata.obsm[k]
for k in list(adata.uns.keys()):
del adata.uns[k]
sc.pp.pca(adata, zero_center=None, n_comps=min(adata.n_vars - 1, 50), **pca_options)
sc.pp.neighbors(adata, **neighbors_options)
sc.tl.umap(adata, **umap_options)
umap = adata.obsm["X_umap"]
result = np.full((obs_mask.shape[0], umap.shape[1]), np.NaN)
result[obs_mask] = umap
return result
backend/czi_hosted/data_anndata/anndata_adaptor.py
+1 -26
View File
@@ -1,20 +1,17 @@
import warnings
from datetime import datetime
import anndata
import numpy as np
from packaging import version
from pandas.core.dtypes.dtypes import CategoricalDtype
from scipy import sparse
from server_timing import Timing as ServerTiming
import backend.common.compute.diffexp_generic as diffexp_generic
from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
from backend.common.constants import Axis, MAX_LAYOUTS
from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata
from backend.common.errors import PrepareError, DatasetAccessError, FilterError
from backend.common.errors import PrepareError, DatasetAccessError
from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
from backend.czi_hosted.compute.scanpy import scanpy_umap
from backend.czi_hosted.data_common.data_adaptor import DataAdaptor
from backend.common.fbs.matrix import encode_matrix_fbs
@@ -301,28 +298,6 @@ class AnndataAdaptor(DataAdaptor):
full_embedding = self.data.obsm[f"X_{ename}"]
return full_embedding[:, 0:dims]
def compute_embedding(self, method, obsFilter):
if Axis.VAR in obsFilter:
raise FilterError("Observation filters may not contain variable conditions")
if method != "umap":
raise NotImplementedError(f"re-embedding method {method} is not available.")
try:
shape = self.get_shape()
obs_mask = self._axis_filter_to_mask(Axis.OBS, obsFilter["obs"], shape[0])
except (KeyError, IndexError):
raise FilterError("Error parsing filter")
with ServerTiming.time("layout.compute"):
X_umap = scanpy_umap(self.data, obs_mask)
# Server picks reemedding name, which must not collide with any other
# embedding name generated by this backend.
name = f"reembed:{method}_{datetime.now().isoformat(timespec='milliseconds')}"
dims = [f"{name}_0", f"{name}_1"]
layout_schema = {"name": name, "type": "float32", "dims": dims}
self.schema["layout"]["obs"].append(layout_schema)
self.data.obsm[f"X_{name}"] = X_umap
return layout_schema
def compute_diffexp_ttest(self, maskA, maskB, top_n=None, lfc_cutoff=None):
if top_n is None:
top_n = self.dataset_config.diffexp__top_n
backend/czi_hosted/data_common/data_adaptor.py
-5
View File
@@ -71,11 +71,6 @@ class DataAdaptor(metaclass=ABCMeta):
"""return an numpy array for the given pre-computed embedding name."""
pass
@abstractmethod
def compute_embedding(self, method, filter):
"""compute a new embedding on the specified obs subset, and return the embedding schema. """
pass
@abstractmethod
def get_X_array(self, obs_mask=None, var_mask=None):
"""return the X array, possibly filtered by obs_mask or var_mask.
backend/czi_hosted/data_cxg/cxg_adaptor.py
-3
View File
@@ -199,9 +199,6 @@ class CxgAdaptor(DataAdaptor):
array = self.open_array(f"emb/{ename}")
return array[:, 0:dims]
def compute_embedding(self, method, filter):
raise NotImplementedError("CXG does not yet support re-embedding")
def compute_diffexp_ttest(self, maskA, maskB, top_n=None, lfc_cutoff=None):
if top_n is None:
top_n = self.dataset_config.diffexp__top_n
backend/czi_hosted/default_config.py
-1
View File
@@ -197,7 +197,6 @@ dataset:
embeddings:
names : []
enable_reembedding: false
diffexp:
enable: true
backend/czi_hosted/eb/app.py
-1
View File
@@ -164,7 +164,6 @@ try:
app_config.update_server_config(multi_dataset__dataroot=dataroot)
# overwrite configuration for the eb app
app_config.update_default_dataset_config(embeddings__enable_reembedding=False,)
app_config.update_server_config(multi_dataset__allowed_matrix_types=["cxg"],)
# complete config
backend/czi_hosted/requirements-dev.txt
+1
View File
@@ -8,4 +8,5 @@ pytest>=3.6.3
python-jose>=3.2.0
twine>=1.12.1
-r requirements.txt
-r requirements-prepare.txt
rsa>=4.7 # not directly required, pinned by Snyk to avoid a vulnerability
backend/czi_hosted/requirements-prepare.txt
+2
View File
@@ -1,2 +1,4 @@
python-igraph
louvain>=0.6
scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here
umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn
backend/czi_hosted/requirements.txt
-2
View File
@@ -21,6 +21,4 @@ scipy>=1.0
requests>=2.22.0
tiledb>=0.5.9,>=0.6.2,!=0.7.2, !=0.8.6
s3fs==0.4.2
scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here
sqlalchemy>=1.3.18
umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn
backend/server/app/app.py
-5
View File
@@ -187,11 +187,6 @@ class LayoutObsAPI(Resource):
def get(self, data_adaptor):
return common_rest.layout_obs_get(request, data_adaptor)
@cache_control(no_store=True)
@rest_get_data_adaptor
def put(self, data_adaptor):
return common_rest.layout_obs_put(request, data_adaptor)
class GenesetsAPI(Resource):
@cache_control(public=True, max_age=ONE_WEEK)
backend/server/cli/launch.py
-10
View File
@@ -107,14 +107,6 @@ def config_args(func):
metavar="<text>",
help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all.",
)
@click.option(
"--experimental-enable-reembedding",
is_flag=True,
default=DEFAULT_CONFIG.dataset_config.embeddings__enable_reembedding,
show_default=False,
hidden=True,
help="Enable experimental on-demand re-embedding using UMAP. WARNING: may be very slow.",
)
@functools.wraps(func)
def wrapper(*args, **kwargs):
return func(*args, **kwargs)
@@ -324,7 +316,6 @@ def launch(
disable_gene_sets_save,
backed,
disable_diffexp,
experimental_enable_reembedding,
config_file,
dump_default_config,
):
@@ -383,7 +374,6 @@ def launch(
presentation__max_categories=max_category_items,
presentation__custom_colors=not disable_custom_colors,
embeddings__names=embedding,
embeddings__enable_reembedding=experimental_enable_reembedding,
diffexp__enable=not disable_diffexp,
diffexp__lfc_cutoff=diffexp_lfc_cutoff,
)
backend/server/common/config/client_config.py
-1
View File
@@ -40,7 +40,6 @@ def get_client_config(app_config, data_adaptor):
"diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
"backed": server_config.adaptor__anndata_adaptor__backed,
"disable-diffexp": not dataset_config.diffexp__enable,
"enable-reembedding": dataset_config.embeddings__enable_reembedding,
"annotations": False,
"annotations_file": None,
"annotations_dir": None,
backend/server/common/config/dataset_config.py
-15
View File
@@ -4,7 +4,6 @@ from os.path import splitext, isdir
from backend.server.common.annotations.local_file_csv import AnnotationsLocalFile
from backend.server.common.config.base_config import BaseConfig
from backend.common.errors import ConfigurationError, AnnotationsError
from backend.server.compute.scanpy import get_scanpy_module
from backend.server.data_common.matrix_loader import MatrixDataLoader
@@ -34,7 +33,6 @@ class DatasetConfig(BaseConfig):
]["gene_sets_file"]
self.embeddings__names = default_config["embeddings"]["names"]
self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"]
self.diffexp__enable = default_config["diffexp"]["enable"]
self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
@@ -173,19 +171,6 @@ class DatasetConfig(BaseConfig):
def handle_embeddings(self):
self.validate_correct_type_of_configuration_attribute("embeddings__names", list)
self.validate_correct_type_of_configuration_attribute("embeddings__enable_reembedding", bool)
server_config = self.app_config.server_config
if self.embeddings__enable_reembedding:
if server_config.single_dataset__datapath:
if server_config.adaptor__anndata_adaptor__backed:
raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
try:
get_scanpy_module()
except NotImplementedError:
# Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported
raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
def handle_diffexp(self, context):
self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool)
backend/server/common/rest.py
-19
View File
@@ -311,25 +311,6 @@ def layout_obs_get(request, data_adaptor):
)
def layout_obs_put(request, data_adaptor):
if not data_adaptor.dataset_config.embeddings__enable_reembedding:
return abort(HTTPStatus.NOT_IMPLEMENTED)
args = request.get_json()
filter = args["filter"] if args else None
if not filter:
return abort_and_log(HTTPStatus.BAD_REQUEST, "obs filter is required")
method = args["method"] if args else "umap"
try:
schema = data_adaptor.compute_embedding(method, filter)
return make_response(jsonify(schema), HTTPStatus.OK, {"Content-Type": "application/json"})
except NotImplementedError as e:
return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, str(e))
except (ValueError, DisabledFeatureError, FilterError) as e:
return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
def genesets_get(request, data_adaptor):
preferred_mimetype = request.accept_mimetypes.best_match(["application/json", "text/csv"])
if preferred_mimetype not in ("application/json", "text/csv"):
backend/server/compute/scanpy.py
-53
View File
@@ -1,53 +0,0 @@
import importlib
import numpy as np
"""
Wrapper for various scanpy modules. Will raise NotImplementedError if the scanpy
module is not installed/available
"""
def get_scanpy_module():
try:
sc = importlib.import_module("scanpy")
# Future: we could enforce versions here, eg, lookat sc.__version__
return sc
except ModuleNotFoundError as e:
raise NotImplementedError("Please install scanpy to enable UMAP re-embedding") from e
except Exception as e:
# will capture other ImportError corner cases
raise NotImplementedError() from e
def scanpy_umap(adata, obs_mask=None, pca_options={}, neighbors_options={}, umap_options={}):
"""
Given adata and an obs mask, return a new embedding for adata[obs_mask, :]
as an ndarray of shape (len(obs_mask), N), where N>=2.
Do NOT mutate adata.
"""
# backed mode is incompatible with the current implementation
if adata.isbacked:
raise NotImplementedError("Backed mode is incompatible with re-embedding")
# safely get scanpy module, which may not be present.
sc = get_scanpy_module()
# https://github.com/theislab/anndata/issues/311
obs_mask = slice(None) if obs_mask is None else obs_mask
adata = adata[obs_mask, :].copy()
for k in list(adata.obsm.keys()):
del adata.obsm[k]
for k in list(adata.uns.keys()):
del adata.uns[k]
sc.pp.pca(adata, zero_center=None, n_comps=min(adata.n_vars - 1, 50), **pca_options)
sc.pp.neighbors(adata, **neighbors_options)
sc.tl.umap(adata, **umap_options)
umap = adata.obsm["X_umap"]
result = np.full((obs_mask.shape[0], umap.shape[1]), np.NaN)
result[obs_mask] = umap
return result
backend/server/data_anndata/anndata_adaptor.py
+1 -26
View File
@@ -1,20 +1,17 @@
import warnings
from datetime import datetime
import anndata
import numpy as np
from packaging import version
from pandas.core.dtypes.dtypes import CategoricalDtype
from scipy import sparse
from server_timing import Timing as ServerTiming
import backend.common.compute.diffexp_generic as diffexp_generic
from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
from backend.common.constants import Axis, MAX_LAYOUTS
from backend.server.common.corpora import corpora_get_props_from_anndata
from backend.common.errors import PrepareError, DatasetAccessError, FilterError
from backend.common.errors import PrepareError, DatasetAccessError
from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
from backend.server.compute.scanpy import scanpy_umap
from backend.server.data_common.data_adaptor import DataAdaptor
from backend.common.fbs.matrix import encode_matrix_fbs
@@ -301,28 +298,6 @@ class AnndataAdaptor(DataAdaptor):
full_embedding = self.data.obsm[f"X_{ename}"]
return full_embedding[:, 0:dims]
def compute_embedding(self, method, obsFilter):
if Axis.VAR in obsFilter:
raise FilterError("Observation filters may not contain variable conditions")
if method != "umap":
raise NotImplementedError(f"re-embedding method {method} is not available.")
try:
shape = self.get_shape()
obs_mask = self._axis_filter_to_mask(Axis.OBS, obsFilter["obs"], shape[0])
except (KeyError, IndexError):
raise FilterError("Error parsing filter")
with ServerTiming.time("layout.compute"):
X_umap = scanpy_umap(self.data, obs_mask)
# Server picks reemedding name, which must not collide with any other
# embedding name generated by this backend.
name = f"reembed:{method}_{datetime.now().isoformat(timespec='milliseconds')}"
dims = [f"{name}_0", f"{name}_1"]
layout_schema = {"name": name, "type": "float32", "dims": dims}
self.schema["layout"]["obs"].append(layout_schema)
self.data.obsm[f"X_{name}"] = X_umap
return layout_schema
def compute_diffexp_ttest(self, maskA, maskB, top_n=None, lfc_cutoff=None):
if top_n is None:
top_n = self.dataset_config.diffexp__top_n
backend/server/data_common/data_adaptor.py
-5
View File
@@ -66,11 +66,6 @@ class DataAdaptor(metaclass=ABCMeta):
"""return an numpy array for the given pre-computed embedding name."""
pass
@abstractmethod
def compute_embedding(self, method, filter):
"""compute a new embedding on the specified obs subset, and return the embedding schema."""
pass
@abstractmethod
def get_X_array(self, obs_mask=None, var_mask=None):
"""return the X array, possibly filtered by obs_mask or var_mask.
backend/server/default_config.py
-1
View File
@@ -71,7 +71,6 @@ dataset:
embeddings:
names : []
enable_reembedding: false
diffexp:
enable: true
backend/server/requirements-dev.txt
+1
View File
@@ -8,3 +8,4 @@ pytest>=3.6.3
python-jose>=3.2.0
twine>=1.12.1
-r requirements.txt
-r requirements-prepare.txt
backend/server/requirements-prepare.txt
+2
View File
@@ -1,2 +1,4 @@
python-igraph>=0.8
louvain>=0.6
scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here
umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn
backend/server/requirements.txt
-2
View File
@@ -21,5 +21,3 @@ PyYAML>=5.4 # CVE-2020-14343
scipy>=1.4
requests>=2.22.0
s3fs==0.4.2
scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here
umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn
backend/test/fixtures/czi_hosted_dataset_config_outline.py
Vendored
-1
View File
@@ -25,7 +25,6 @@ dataset:
embeddings:
names: {embedding_names}
enable_reembedding: {enable_reembedding}
diffexp:
enable: {enable_difexp}
backend/test/fixtures/dataset_config_outline.py
Vendored
-1
View File
@@ -22,7 +22,6 @@ dataset:
embeddings:
names: {embedding_names}
enable_reembedding: {enable_reembedding}
diffexp:
enable: {enable_difexp}
backend/test/test_czi_hosted/unit/__init__.py
-1
View File
@@ -170,7 +170,6 @@ class BaseTest(unittest.TestCase):
multi_dataset__index=True,
multi_dataset__allowed_matrix_types=["cxg"]
)
app_config.update_default_dataset_config(embeddings__enable_reembedding=False, )
app_config.complete_config(logging.info)
app = TestServer(app_config).app
backend/test/test_czi_hosted/unit/common/config/__init__.py
-3
View File
@@ -125,7 +125,6 @@ class ConfigTests(BaseTest):
local_file_csv_directory="null",
local_file_csv_file="null",
embedding_names=[],
enable_reembedding="false",
enable_difexp="true",
lfc_cutoff=0.01,
top_n=10,
@@ -192,7 +191,6 @@ class ConfigTests(BaseTest):
local_file_csv_directory=local_file_csv_directory,
local_file_csv_file=local_file_csv_file,
embedding_names=embedding_names,
enable_reembedding=enable_reembedding,
enable_difexp=enable_difexp,
lfc_cutoff=lfc_cutoff,
top_n=top_n,
@@ -228,7 +226,6 @@ class ConfigTests(BaseTest):
local_file_csv_directory="null",
local_file_csv_file="null",
embedding_names=[],
enable_reembedding="false",
enable_difexp="true",
lfc_cutoff=0.01,
top_n=10,
backend/test/test_czi_hosted/unit/common/config/test_dataset_config.py
+1 -15
View File
@@ -49,7 +49,7 @@ class TestDatasetConfig(ConfigTests):
def test_complete_config_checks_all_attr(self, mock_check_attrs):
mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
self.dataset_config.complete_config(self.context)
self.assertEqual(mock_check_attrs.call_count, 19)
self.assertEqual(mock_check_attrs.call_count, 18)
def test_app_sets_script_vars(self):
config = self.get_config(scripts=["path/to/script"])
@@ -130,20 +130,6 @@ class TestDatasetConfig(ConfigTests):
cwd = os.getcwd()
self.assertEqual(config.default_dataset_config.user_annotations._get_output_dir(), cwd)
def test_handle_embeddings__checks_data_file_types(self):
file_name = self.custom_app_config(
embedding_names=["name1", "name2"],
enable_reembedding="true",
dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
anndata_backed="true",
config_file_name=self.config_file_name,
)
config = AppConfig()
config.update_from_config_file(file_name)
config.server_config.complete_config(self.context)
with self.assertRaises(ConfigurationError):
config.default_dataset_config.handle_embeddings()
def test_handle_diffexp__raises_warning_for_large_datasets(self):
config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15)
config.server_config.complete_config(self.context)
backend/test/test_czi_hosted/unit/common/test_api.py
+1 -30
View File
@@ -69,35 +69,6 @@ class EndPoints(BaseTest):
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
def test_put_layout_fbs(self):
# first check that re-embedding is turned on
self.app.auth.get_user_id = lambda : "123"
result = self.client.get(f"{self.TEST_URL_BASE}config")
config_data = json.loads(result.data)
re_embed = config_data["config"]["parameters"]["enable-reembedding"]
if not re_embed:
return
# attempt to reembed with umap over 100 cells.
endpoint = "layout/obs"
url = f"{self.TEST_URL_BASE}{endpoint}"
data = {}
data["filter"] = {}
data["filter"]["obs"] = {}
data["filter"]["obs"]["index"] = list(range(100))
data["method"] = "umap"
result = self.client.put(url, json=data)
self.assertEqual(result.status_code, HTTPStatus.OK)
result_data = result.json()
self.assertIsInstance(result_data, dict)
self.assertEqual(result_data["type"], "float32")
self.assertTrue(result_data["name"].startswith("reembed:umap_"))
self.assertIsInstance(result_data["dims"], list)
self.assertEqual(len(result_data["dims"]), 2)
dims = result_data["dims"]
self.assertTrue(dims[0].startswith("reembed:umap_") and dims[0].endswith("_0"))
self.assertTrue(dims[1].startswith("reembed:umap_") and dims[1].endswith("_1"))
def test_bad_filter(self):
endpoint = "data/var"
url = f"{self.TEST_URL_BASE}{endpoint}"
@@ -421,7 +392,7 @@ class EndPointsCxg(EndPoints):
@classmethod
def setUpClass(cls):
app_config = AppConfig()
app_config.update_default_dataset_config(embeddings__enable_reembedding=True, user_annotations__enable=False)
app_config.update_default_dataset_config(user_annotations__enable=False)
def test_get_genesets_json(self):
self.app.auth.is_user_authenticated = lambda: True
backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor.py
-36
View File
@@ -196,39 +196,3 @@ class AdaptorTest(unittest.TestCase):
self.assertEqual(data["n_rows"], 2638)
self.assertEqual(data["n_cols"], 3)
self.assertTrue((data["col_idx"] == [15, 1818, 1837]).all())
def test_compute_embedding(self):
filter = {"obs": {"index": [[0, 100]]}}
# Verify that we correctly handle the case where we lack scanpy
import unittest.mock
with unittest.mock.patch.dict(sys.modules, {"scanpy": None}):
with self.assertRaises(NotImplementedError):
self.data.compute_embedding("umap", filter)
# if we happen to have scanpy, test the full API, else punt
import importlib
scanpy_spec = importlib.util.find_spec("scanpy")
if scanpy_spec is None:
print("Skipping compute_embedding test as ScanPy not installed")
return
# this feature is unsupported in backed mode, and we expect an error
if self.data.data.isbacked:
with self.assertRaises(NotImplementedError):
self.data.compute_embedding("umap", filter)
return
schema = self.data.compute_embedding("umap", filter)
self.assertIsInstance(schema["name"], str)
name = schema["name"]
self.assertEqual(schema["type"], "float32")
self.assertEqual(schema["dims"], [f"{name}_0", f"{name}_1"])
emb = self.data.data.obsm[f"X_{name}"]
self.assertEqual(emb.shape, (2638, 2))
self.assertTrue(np.isfinite(emb[0:100]).all())
self.assertTrue(np.isnan(emb[100:]).all())
backend/test/test_server/unit/common/config/__init__.py
-3
View File
@@ -97,7 +97,6 @@ class ConfigTests(unittest.TestCase):
local_file_csv_gene_sets_file="null",
gene_sets_readonly="false",
embedding_names=[],
enable_reembedding="false",
enable_difexp="true",
lfc_cutoff=0.01,
top_n=10,
@@ -148,7 +147,6 @@ class ConfigTests(unittest.TestCase):
local_file_csv_gene_sets_file=local_file_csv_gene_sets_file,
gene_sets_readonly=gene_sets_readonly,
embedding_names=embedding_names,
enable_reembedding=enable_reembedding,
enable_difexp=enable_difexp,
lfc_cutoff=lfc_cutoff,
top_n=top_n,
@@ -184,7 +182,6 @@ class ConfigTests(unittest.TestCase):
local_file_csv_gene_sets_file="null",
gene_sets_readonly="false",
embedding_names=[],
enable_reembedding="false",
enable_difexp="true",
lfc_cutoff=0.01,
top_n=10,
backend/test/test_server/unit/common/config/test_dataset_config.py
+1 -15
View File
@@ -46,7 +46,7 @@ class TestDatasetConfig(ConfigTests):
mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
self.dataset_config.complete_config(self.context)
self.assertIsNotNone(self.config.server_config.data_adaptor)
self.assertEqual(mock_check_attrs.call_count, 17)
self.assertEqual(mock_check_attrs.call_count, 16)
def test_app_sets_script_vars(self):
config = self.get_config(scripts=["path/to/script"])
@@ -106,20 +106,6 @@ class TestDatasetConfig(ConfigTests):
cwd = os.getcwd()
self.assertEqual(config.dataset_config.user_annotations._get_output_dir(), cwd)
def test_handle_embeddings__checks_data_file_types(self):
file_name = self.custom_app_config(
embedding_names=["name1", "name2"],
enable_reembedding="true",
dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
anndata_backed="true",
config_file_name=self.config_file_name,
)
config = AppConfig()
config.update_from_config_file(file_name)
config.server_config.complete_config(self.context)
with self.assertRaises(ConfigurationError):
config.dataset_config.handle_embeddings()
def test_handle_diffexp__raises_warning_for_large_datasets(self):
config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15)
config.server_config.complete_config(self.context)
backend/test/test_server/unit/common/test_api.py
-29
View File
@@ -73,34 +73,6 @@ class EndPoints(object):
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
def test_put_layout_fbs(self):
# first check that re-embedding is turned on
result = self.session.get(f"{self.URL_BASE}config")
config_data = result.json()
re_embed = config_data["config"]["parameters"]["enable-reembedding"]
if not re_embed:
return
# attempt to reembed with umap over 100 cells.
endpoint = "layout/obs"
url = f"{self.URL_BASE}{endpoint}"
data = {}
data["filter"] = {}
data["filter"]["obs"] = {}
data["filter"]["obs"]["index"] = list(range(100))
data["method"] = "umap"
result = self.session.put(url, json=data)
self.assertEqual(result.status_code, HTTPStatus.OK)
result_data = result.json()
self.assertIsInstance(result_data, dict)
self.assertEqual(result_data["type"], "float32")
self.assertTrue(result_data["name"].startswith("reembed:umap_"))
self.assertIsInstance(result_data["dims"], list)
self.assertEqual(len(result_data["dims"]), 2)
dims = result_data["dims"]
self.assertTrue(dims[0].startswith("reembed:umap_") and dims[0].endswith("_0"))
self.assertTrue(dims[1].startswith("reembed:umap_") and dims[1].endswith("_1"))
def test_bad_filter(self):
endpoint = "data/var"
url = f"{self.URL_BASE}{endpoint}"
@@ -389,7 +361,6 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
"--disable-annotations",
"--disable-gene-sets-save",
"--experimental-enable-reembedding",
],
)
backend/test/test_server/unit/data_anndata/test_anndata_adaptor.py
-36
View File
@@ -198,39 +198,3 @@ class AdaptorTest(unittest.TestCase):
self.assertEqual(data["n_rows"], 2638)
self.assertEqual(data["n_cols"], 3)
self.assertTrue((data["col_idx"] == [15, 1818, 1837]).all())
def test_compute_embedding(self):
filter = {"obs": {"index": [[0, 100]]}}
# Verify that we correctly handle the case where we lack scanpy
import unittest.mock
with unittest.mock.patch.dict(sys.modules, {"scanpy": None}):
with self.assertRaises(NotImplementedError):
self.data.compute_embedding("umap", filter)
# if we happen to have scanpy, test the full API, else punt
import importlib
scanpy_spec = importlib.util.find_spec("scanpy")
if scanpy_spec is None:
print("Skipping compute_embedding test as ScanPy not installed")
return
# this feature is unsupported in backed mode, and we expect an error
if self.data.data.isbacked:
with self.assertRaises(NotImplementedError):
self.data.compute_embedding("umap", filter)
return
schema = self.data.compute_embedding("umap", filter)
self.assertIsInstance(schema["name"], str)
name = schema["name"]
self.assertEqual(schema["type"], "float32")
self.assertEqual(schema["dims"], [f"{name}_0", f"{name}_1"])
emb = self.data.data.obsm[f"X_{name}"]
self.assertEqual(emb.shape, (2638, 2))
self.assertTrue(np.isfinite(emb[0:100]).all())
self.assertTrue(np.isnan(emb[100:]).all())
client/__tests__/e2e/test_config.yaml
-1
View File
@@ -25,4 +25,3 @@ dataset:
embeddings:
names: []
enable_reembedding: false
client/src/actions/embedding.js
+1 -1
View File
@@ -11,7 +11,7 @@ export async function _switchEmbedding(
newEmbeddingName
) {
/*
DRY helper used by this and reembedding action creators
DRY helper used by embedding action creators
*/
const base = prevAnnoMatrix.base();
const embeddingDf = await base.fetch("emb", newEmbeddingName);
client/src/actions/index.js
-4
View File
@@ -5,9 +5,6 @@ import {
doJsonRequest,
dispatchNetworkErrorMessageToUser,
} from "../util/actionHelpers";
import {
requestReembed /* , reembedResetWorldToUniverse -- disabled temporarily, TODO issue #1606 */,
} from "./reembed";
import { loadUserColorConfig } from "../util/stateManager/colorHelpers";
import * as selnActions from "./selection";
import * as annoActions from "./annotation";
@@ -243,7 +240,6 @@ export default {
requestDifferentialExpression,
requestSingleGeneExpressionCountsForColoringPOST,
requestUserDefinedGene,
requestReembed,
selectContinuousMetadataAction: selnActions.selectContinuousMetadataAction,
selectCategoricalMetadataAction: selnActions.selectCategoricalMetadataAction,
selectCategoricalAllMetadataAction:
client/src/actions/reembed.js
-112
View File
@@ -1,112 +0,0 @@
import { API } from "../globals";
import {
postNetworkErrorToast,
postAsyncSuccessToast,
postAsyncFailureToast,
} from "../components/framework/toasters";
import { _switchEmbedding } from "./embedding";
function abortableFetch(request, opts, timeout = 0) {
const controller = new AbortController();
const { signal } = controller;
return {
abort: () => controller.abort(),
isAborted: () => signal.aborted,
ready: () => {
if (timeout) {
setTimeout(() => controller.abort(), timeout);
}
return fetch(request, { ...opts, signal });
},
};
}
async function doReembedFetch(dispatch, getState) {
const state = getState();
let cells = state.annoMatrix.rowIndex.labels();
// These lines ensure that we convert any TypedArray to an Array.
// This is necessary because JSON.stringify() does some very strange
// things with TypedArrays (they are marshalled to JSON objects, rather
// than being marshalled as a JSON array).
cells = Array.isArray(cells) ? cells : Array.from(cells);
const af = abortableFetch(
`${API.prefix}${API.version}layout/obs`,
{
method: "PUT",
headers: new Headers({
Accept: "application/octet-stream",
"Content-Type": "application/json",
}),
body: JSON.stringify({
method: "umap",
filter: { obs: { index: cells } },
}),
credentials: "include",
},
60000 // 1 minute timeout
);
dispatch({
type: "reembed: request start",
abortableFetch: af,
});
const res = await af.ready();
if (res.ok && res.headers.get("Content-Type").includes("application/json")) {
return res;
}
// else an error
let msg = `Unexpected HTTP response ${res.status}, ${res.statusText}`;
const body = await res.text();
if (body && body.length > 0) {
msg = `${msg} -- ${body}`;
}
throw new Error(msg);
}
/*
functions below are dispatch-able
*/
export function requestReembed() {
return async (dispatch, getState) => {
try {
const res = await doReembedFetch(dispatch, getState);
const schema = await res.json();
dispatch({
type: "reembed: request completed",
});
const {
annoMatrix: prevAnnoMatrix,
obsCrossfilter: prevCrossfilter,
} = getState();
const base = prevAnnoMatrix.base().addEmbedding(schema);
const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
base,
prevCrossfilter,
schema.name
);
dispatch({
type: "reembed: add reembedding",
schema,
annoMatrix,
obsCrossfilter,
});
postAsyncSuccessToast("Re-embedding has completed.");
} catch (error) {
dispatch({
type: "reembed: request aborted",
});
if (error.name === "AbortError") {
postAsyncFailureToast("Re-embedding calculation was aborted.");
} else {
postNetworkErrorToast(`Re-embedding: ${error.message}`);
}
console.log("Reembed exception:", error, error.name, error.message);
}
};
}
client/src/components/menubar/index.js
-5
View File
@@ -11,7 +11,6 @@ import AuthButtons from "./authButtons";
import Subset from "./subset";
import UndoRedoReset from "./undoRedo";
import DiffexpButtons from "./diffexpButtons";
import Reembedding from "./reembedding";
import { getEmbSubsetView } from "../../util/stateManager/viewStackHelpers";
@connect((state) => {
@@ -49,8 +48,6 @@ import { getEmbSubsetView } from "../../util/stateManager/viewStackHelpers";
tosURL: state.config?.parameters?.about_legal_tos,
privacyURL: state.config?.parameters?.about_legal_privacy,
categoricalSelection: state.categoricalSelection,
enableReembedding:
state.config?.parameters?.["enable-reembedding"] ?? false,
};
})
class MenuBar extends React.PureComponent {
@@ -212,7 +209,6 @@ class MenuBar extends React.PureComponent {
colorAccessor,
subsetPossible,
subsetResetPossible,
enableReembedding,
userInfo,
auth,
} = this.props;
@@ -262,7 +258,6 @@ class MenuBar extends React.PureComponent {
this.handleClipPercentileMinValueChange
}
/>
{enableReembedding ? <Reembedding /> : null}
<Tooltip
content="When a category is colored by, show labels on the graph"
position="bottom"
client/src/components/menubar/reembedding.js
-40
View File
@@ -1,40 +0,0 @@
import React from "react";
import { connect } from "react-redux";
import { AnchorButton, ButtonGroup, Tooltip } from "@blueprintjs/core";
import * as globals from "../../globals";
import actions from "../../actions";
import styles from "./menubar.css";
@connect((state) => ({
reembedController: state.reembedController,
annoMatrix: state.annoMatrix,
}))
class Reembedding extends React.PureComponent {
render() {
const { dispatch, annoMatrix, reembedController } = this.props;
const loading = !!reembedController?.pendingFetch;
const disabled = annoMatrix.nObs === annoMatrix.schema.dataframe.nObs;
const tipContent = disabled
? "Subset cells first, then click to recompute UMAP embedding."
: "Click to recompute UMAP embedding on the current cell subset.";
return (
<ButtonGroup className={styles.menubarButton}>
<Tooltip
content={tipContent}
position="bottom"
hoverOpenDelay={globals.tooltipHoverOpenDelay}
>
<AnchorButton
icon="new-object"
disabled={disabled}
onClick={() => dispatch(actions.requestReembed())}
loading={loading}
/>
</Tooltip>
</ButtonGroup>
);
}
}
export default Reembedding;
client/src/reducers/index.js
-2
View File
@@ -20,7 +20,6 @@ import genesetsUI from "./genesetsUI";
import autosave from "./autosave";
import centroidLabels from "./centroidLabels";
import pointDialation from "./pointDilation";
import { reembedController } from "./reembed";
import { gcMiddleware as annoMatrixGC } from "../annoMatrix";
import undoableConfig from "./undoableConfig";
@@ -42,7 +41,6 @@ const Reducer = undoable(
["differential", differential],
["centroidLabels", centroidLabels],
["pointDilation", pointDialation],
["reembedController", reembedController],
["autosave", autosave],
["userInfo", userInfo],
]),
client/src/reducers/layoutChoice.js
-13
View File
@@ -47,19 +47,6 @@ const LayoutChoice = (
return { ...state, current, currentDimNames };
}
case "reembed: add reembedding": {
const { schema } = nextSharedState.annoMatrix;
const { name } = action.schema;
const available = Array.from(new Set(state.available).add(name));
const currentDimNames = schema.layout.obsByName[name].dims;
return {
...state,
available,
current: name,
currentDimNames,
};
}
default: {
return state;
}
client/src/reducers/reembed.js
-29
View File
@@ -1,29 +0,0 @@
/*
controller state is not part of the undo/redo history
*/
export const reembedController = (
state = {
pendingFetch: null,
},
action
) => {
switch (action.type) {
case "reembed: request start": {
return {
...state,
pendingFetch: action.abortableFetch,
};
}
case "reembed: request aborted":
case "reembed: request cancel":
case "reembed: request completed": {
return {
...state,
pendingFetch: null,
};
}
default: {
return state;
}
}
};