fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-10-08 05:48:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

remove experimental reembedding support (#2301)

Browse Source 
* remove experimental reembedding support

* lint

* lint

* add prepare requirements to requirements-dev

* oops, revert accidental deletion of import

* more test modifications

* remove obsolete unit tests
This commit is contained in:
Bruce Martin
2021-07-15 13:55:26 -07:00
committed by GitHub
parent e334fbe96e
commit 0667ad0274
46 changed files with 12 additions and 661 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/czi_hosted/app/app.py
-5
View File
@@ -312,11 +312,6 @@ class LayoutObsAPI(DatasetResource):
def get(self, data_adaptor):
return common_rest.layout_obs_get(request, data_adaptor)
@cache_control(no_store=True)
@rest_get_data_adaptor
def put(self, data_adaptor):
return common_rest.layout_obs_put(request, data_adaptor)
class GenesetsAPI(DatasetResource):
@cache_control(public=True, max_age=ONE_WEEK)
backend/czi_hosted/cli/launch.py
-10
View File
@@ -90,14 +90,6 @@ def config_args(func):
metavar="<text>",
help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all.",
)
@click.option(
"--experimental-enable-reembedding",
is_flag=True,
default=DEFAULT_CONFIG.default_dataset_config.embeddings__enable_reembedding,
show_default=False,
hidden=True,
help="Enable experimental on-demand re-embedding using UMAP. WARNING: may be very slow.",
)
@functools.wraps(func)
def wrapper(*args, **kwargs):
return func(*args, **kwargs)
@@ -314,7 +306,6 @@ def launch(
annotations_dir,
backed,
disable_diffexp,
experimental_enable_reembedding,
config_file,
dump_default_config,
):
@@ -376,7 +367,6 @@ def launch(
presentation__max_categories=max_category_items,
presentation__custom_colors=not disable_custom_colors,
embeddings__names=embedding,
embeddings__enable_reembedding=experimental_enable_reembedding,
diffexp__enable=not disable_diffexp,
diffexp__lfc_cutoff=diffexp_lfc_cutoff,
)
backend/czi_hosted/common/config/client_config.py
-1
View File
@@ -40,7 +40,6 @@ def get_client_config(app_config, data_adaptor):
"diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
"backed": server_config.adaptor__anndata_adaptor__backed,
"disable-diffexp": not dataset_config.diffexp__enable,
"enable-reembedding": dataset_config.embeddings__enable_reembedding,
"annotations": False,
"annotations_file": None,
"annotations_dir": None,
backend/czi_hosted/common/config/dataset_config.py
-21
View File
@@ -6,8 +6,6 @@ from backend.czi_hosted.common.annotations.hosted_tiledb import AnnotationsHoste
from backend.czi_hosted.common.annotations.local_file_csv import AnnotationsLocalFile
from backend.czi_hosted.common.config.base_config import BaseConfig
from backend.common.errors import ConfigurationError
from backend.czi_hosted.compute.scanpy import get_scanpy_module
from backend.czi_hosted.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
from backend.czi_hosted.db.db_utils import DbUtils
@@ -41,7 +39,6 @@ class DatasetConfig(BaseConfig):
]["hosted_file_directory"]
self.embeddings__names = default_config["embeddings"]["names"]
self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"]
self.diffexp__enable = default_config["diffexp"]["enable"]
self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
@@ -186,24 +183,6 @@ class DatasetConfig(BaseConfig):
def handle_embeddings(self):
self.validate_correct_type_of_configuration_attribute("embeddings__names", list)
self.validate_correct_type_of_configuration_attribute("embeddings__enable_reembedding", bool)
server_config = self.app_config.server_config
if self.embeddings__enable_reembedding:
if server_config.single_dataset__datapath:
matrix_data_loader = MatrixDataLoader(
server_config.single_dataset__datapath, app_config=self.app_config
)
if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD:
raise ConfigurationError("enable-reembedding is only supported with H5AD files.")
if server_config.adaptor__anndata_adaptor__backed:
raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
try:
get_scanpy_module()
except NotImplementedError:
# Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported
raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
def handle_diffexp(self, context):
self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool)
backend/czi_hosted/common/rest.py
-19
View File
@@ -312,25 +312,6 @@ def layout_obs_get(request, data_adaptor):
)
def layout_obs_put(request, data_adaptor):
if not data_adaptor.dataset_config.embeddings__enable_reembedding:
return abort(HTTPStatus.NOT_IMPLEMENTED)
args = request.get_json()
filter = args["filter"] if args else None
if not filter:
return abort_and_log(HTTPStatus.BAD_REQUEST, "obs filter is required")
method = args["method"] if args else "umap"
try:
schema = data_adaptor.compute_embedding(method, filter)
return make_response(jsonify(schema), HTTPStatus.OK, {"Content-Type": "application/json"})
except NotImplementedError as e:
return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, str(e))
except (ValueError, DisabledFeatureError, FilterError) as e:
return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
def genesets_get(request, data_adaptor):
preferred_mimetype = request.accept_mimetypes.best_match(["application/json", "text/csv"])
if preferred_mimetype not in ("application/json", "text/csv"):
backend/czi_hosted/compute/scanpy.py
-53
View File
@@ -1,53 +0,0 @@
import importlib
import numpy as np
"""
Wrapper for various scanpy modules. Will raise NotImplementedError if the scanpy
module is not installed/available
"""
def get_scanpy_module():
try:
sc = importlib.import_module("scanpy")
# Future: we could enforce versions here, eg, lookat sc.__version__
return sc
except ModuleNotFoundError as e:
raise NotImplementedError("Please install scanpy to enable UMAP re-embedding") from e
except Exception as e:
# will capture other ImportError corner cases
raise NotImplementedError() from e
def scanpy_umap(adata, obs_mask=None, pca_options={}, neighbors_options={}, umap_options={}):
"""
Given adata and an obs mask, return a new embedding for adata[obs_mask, :]
as an ndarray of shape (len(obs_mask), N), where N>=2.
Do NOT mutate adata.
"""
# backed mode is incompatible with the current implementation
if adata.isbacked:
raise NotImplementedError("Backed mode is incompatible with re-embedding")
# safely get scanpy module, which may not be present.
sc = get_scanpy_module()
# https://github.com/theislab/anndata/issues/311
obs_mask = slice(None) if obs_mask is None else obs_mask
adata = adata[obs_mask, :].copy()
for k in list(adata.obsm.keys()):
del adata.obsm[k]
for k in list(adata.uns.keys()):
del adata.uns[k]
sc.pp.pca(adata, zero_center=None, n_comps=min(adata.n_vars - 1, 50), **pca_options)
sc.pp.neighbors(adata, **neighbors_options)
sc.tl.umap(adata, **umap_options)
umap = adata.obsm["X_umap"]
result = np.full((obs_mask.shape[0], umap.shape[1]), np.NaN)
result[obs_mask] = umap
return result
backend/czi_hosted/data_anndata/anndata_adaptor.py
+1 -26
View File
@@ -1,20 +1,17 @@
import warnings
from datetime import datetime
import anndata
import numpy as np
from packaging import version
from pandas.core.dtypes.dtypes import CategoricalDtype
from scipy import sparse
from server_timing import Timing as ServerTiming
import backend.common.compute.diffexp_generic as diffexp_generic
from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
from backend.common.constants import Axis, MAX_LAYOUTS
from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata
from backend.common.errors import PrepareError, DatasetAccessError, FilterError
from backend.common.errors import PrepareError, DatasetAccessError
from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
from backend.czi_hosted.compute.scanpy import scanpy_umap
from backend.czi_hosted.data_common.data_adaptor import DataAdaptor
from backend.common.fbs.matrix import encode_matrix_fbs
@@ -301,28 +298,6 @@ class AnndataAdaptor(DataAdaptor):
full_embedding = self.data.obsm[f"X_{ename}"]
return full_embedding[:, 0:dims]
def compute_embedding(self, method, obsFilter):
if Axis.VAR in obsFilter:
raise FilterError("Observation filters may not contain variable conditions")
if method != "umap":
raise NotImplementedError(f"re-embedding method {method} is not available.")
try:
shape = self.get_shape()
obs_mask = self._axis_filter_to_mask(Axis.OBS, obsFilter["obs"], shape[0])
except (KeyError, IndexError):
raise FilterError("Error parsing filter")
with ServerTiming.time("layout.compute"):
X_umap = scanpy_umap(self.data, obs_mask)
# Server picks reemedding name, which must not collide with any other
# embedding name generated by this backend.
name = f"reembed:{method}_{datetime.now().isoformat(timespec='milliseconds')}"
dims = [f"{name}_0", f"{name}_1"]
layout_schema = {"name": name, "type": "float32", "dims": dims}
self.schema["layout"]["obs"].append(layout_schema)
self.data.obsm[f"X_{name}"] = X_umap
return layout_schema
def compute_diffexp_ttest(self, maskA, maskB, top_n=None, lfc_cutoff=None):
if top_n is None:
top_n = self.dataset_config.diffexp__top_n
backend/czi_hosted/data_common/data_adaptor.py
-5
View File
@@ -71,11 +71,6 @@ class DataAdaptor(metaclass=ABCMeta):
"""return an numpy array for the given pre-computed embedding name."""
pass
@abstractmethod
def compute_embedding(self, method, filter):
"""compute a new embedding on the specified obs subset, and return the embedding schema. """
pass
@abstractmethod
def get_X_array(self, obs_mask=None, var_mask=None):
"""return the X array, possibly filtered by obs_mask or var_mask.
backend/czi_hosted/data_cxg/cxg_adaptor.py
-3
View File
@@ -199,9 +199,6 @@ class CxgAdaptor(DataAdaptor):
array = self.open_array(f"emb/{ename}")
return array[:, 0:dims]
def compute_embedding(self, method, filter):
raise NotImplementedError("CXG does not yet support re-embedding")
def compute_diffexp_ttest(self, maskA, maskB, top_n=None, lfc_cutoff=None):
if top_n is None:
top_n = self.dataset_config.diffexp__top_n
backend/czi_hosted/default_config.py
-1
View File
@@ -197,7 +197,6 @@ dataset:
embeddings:
names : []
enable_reembedding: false
diffexp:
enable: true
backend/czi_hosted/eb/app.py
-1
View File
@@ -164,7 +164,6 @@ try:
app_config.update_server_config(multi_dataset__dataroot=dataroot)
# overwrite configuration for the eb app
app_config.update_default_dataset_config(embeddings__enable_reembedding=False,)
app_config.update_server_config(multi_dataset__allowed_matrix_types=["cxg"],)
# complete config
backend/czi_hosted/requirements-dev.txt
+1
View File
@@ -8,4 +8,5 @@ pytest>=3.6.3
python-jose>=3.2.0
twine>=1.12.1
-r requirements.txt
-r requirements-prepare.txt
rsa>=4.7 # not directly required, pinned by Snyk to avoid a vulnerability
backend/czi_hosted/requirements-prepare.txt
+2
View File
@@ -1,2 +1,4 @@
python-igraph
louvain>=0.6
scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here
umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn
backend/czi_hosted/requirements.txt
-2
View File
@@ -21,6 +21,4 @@ scipy>=1.0
requests>=2.22.0
tiledb>=0.5.9,>=0.6.2,!=0.7.2, !=0.8.6
s3fs==0.4.2
scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here
sqlalchemy>=1.3.18
umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn