From 1ea2b7fe806d7a5230b06479648c6c2febdaa8d8 Mon Sep 17 00:00:00 2001 From: Bruce Martin Date: Fri, 23 Jul 2021 11:36:26 -0700 Subject: [PATCH] fix for incorrect stats computation in diff exp t-test (#2318) * 2211 fixes * lint * lint * add missing test and bug found by test * change terminology for count distribution * update scanpy requirement * update scanpy requirement --- backend/common/compute/diffexp_generic.py | 44 +++++++++---- .../common/compute/estimate_distribution.py | 62 +++++++++++++++++++ backend/common/constants.py | 5 ++ .../common/config/dataset_config.py | 10 +++ .../data_anndata/anndata_adaptor.py | 18 +++++- .../czi_hosted/data_common/data_adaptor.py | 22 +++++-- backend/czi_hosted/data_cxg/cxg_adaptor.py | 10 ++- backend/czi_hosted/default_config.py | 2 + backend/czi_hosted/requirements-prepare.txt | 2 +- backend/czi_hosted/requirements.txt | 4 +- backend/server/cli/launch.py | 10 +++ .../server/common/config/dataset_config.py | 10 +++ .../server/data_anndata/anndata_adaptor.py | 27 +++++++- backend/server/data_common/data_adaptor.py | 6 +- backend/server/default_config.py | 2 + backend/server/requirements-prepare.txt | 2 +- backend/server/requirements.txt | 6 +- .../czi_hosted_dataset_config_outline.py | 2 + .../test/fixtures/dataset_config_outline.py | 2 + .../unit/utils/test_type_conversion_utils.py | 6 +- .../unit/common/config/__init__.py | 3 + .../unit/common/config/test_dataset_config.py | 2 +- .../unit/compute/test_diffexp_cxg.py | 41 ++++++------ .../unit/common/config/__init__.py | 3 + .../unit/common/config/test_dataset_config.py | 4 +- .../unit/compute/test_diffexp_h5ad.py | 40 ++++++------ .../test_server/unit/compute/test_est_dist.py | 41 ++++++++++++ .../unit/data_anndata/test_anndata_adaptor.py | 40 +++++++----- 28 files changed, 336 insertions(+), 90 deletions(-) create mode 100644 backend/common/compute/estimate_distribution.py create mode 100644 backend/test/test_server/unit/compute/test_est_dist.py diff --git a/backend/common/compute/diffexp_generic.py b/backend/common/compute/diffexp_generic.py index be6f7ded..3b2be195 100644 --- a/backend/common/compute/diffexp_generic.py +++ b/backend/common/compute/diffexp_generic.py @@ -1,5 +1,6 @@ import numpy as np from scipy import sparse, stats +from backend.common.constants import XApproxDistribution def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01): @@ -7,7 +8,7 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01): Return differential expression statistics for top N variables. Algorithm: - - compute log fold change (log2(meanA/meanB)) + - compute fold change - compute Welch's t-test statistic and pvalue (w/ Bonferroni correction) - return top N abs(logfoldchange) where lfc > diffexp_lfc_cutoff @@ -26,21 +27,24 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01): :param top_n: number of variables to return stats for :param diffexp_lfc_cutoff: minimum absolute value returning [ varindex, logfoldchange, pval, pval_adj ] for top N genes - :return: for top N genes, {"positive": for top N genes, [ varindex, logfoldchange, pval, pval_adj ], "negative": for top N genes, [ varindex, logfoldchange, pval, pval_adj ]} + :return: for top N genes, {"positive": for top N genes, [ varindex, foldchange, pval, pval_adj ], "negative": for top N genes, [ varindex, foldchange, pval, pval_adj ]} """ + X_approx_distribution = adaptor.get_X_approx_distribution() dataA = adaptor.get_X_array(maskA, None) dataB = adaptor.get_X_array(maskB, None) # mean, variance, N - calculate for both selections - meanA, vA, nA = mean_var_n(dataA) - meanB, vB, nB = mean_var_n(dataB) + meanA, vA, nA = mean_var_n(dataA, X_approx_distribution) + meanB, vB, nB = mean_var_n(dataB, X_approx_distribution) res = diffexp_ttest_from_mean_var(meanA, vA, nA, meanB, vB, nB, top_n, diffexp_lfc_cutoff) return res def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp_lfc_cutoff): + # IMPORTANT NOTE: this code assumes the data is normally distributed and/or already logged. + n_var = meanA.shape[0] top_n = min(top_n, n_var) @@ -64,15 +68,15 @@ def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp pvals_adj = pvals * n_var pvals_adj[pvals_adj > 1] = 1 # cap adjusted p-value at 1 - # logfoldchanges: log2(meanA / meanB) - logfoldchanges = np.log2(np.abs((meanA + 1e-9) / (meanB + 1e-9))) + # log fold change. The data is normally distributed/logged, so just subtract the means. + logfoldchanges = meanA - meanB stats_to_sort = tscores # find all with lfc > cutoff lfc_above_cutoff_idx = np.nonzero(np.abs(logfoldchanges) > diffexp_lfc_cutoff)[0] # derive sort order - if lfc_above_cutoff_idx.shape[0] > top_n*2: + if lfc_above_cutoff_idx.shape[0] > top_n * 2: # partition top N rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], (top_n, -top_n)) rel_t_partition_top_n = np.concatenate((rel_t_partition[-top_n:], rel_t_partition[:top_n])) @@ -95,16 +99,21 @@ def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp pvals_adj_top_n = pvals_adj[sort_order] # varIndex, logfoldchange, pval, pval_adj - result = {"positive": [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in - range(top_n)], - "negative": [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in - range(-1, -1 - top_n, -1)], } + result = { + "positive": [ + [sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n) + ], + "negative": [ + [sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] + for i in range(-1, -1 - top_n, -1) + ], + } return result # Convenience function which handles sparse data -def mean_var_n(X): +def mean_var_n(X, X_approx_distribution=XApproxDistribution.NORMAL): """ Two-pass variance calculation. Numerically (more) stable than naive methods (and same method used by numpy.var()) @@ -122,16 +131,27 @@ def mean_var_n(X): with np.errstate(divide="call", invalid="call", call=fp_err_set): n = X.shape[0] if sparse.issparse(X): + if X_approx_distribution == XApproxDistribution.COUNT: + X = X.log1p() mean = X.mean(axis=0).A1 dfm = X - mean sumsq = np.sum(np.multiply(dfm, dfm), axis=0).A1 v = sumsq / (n - 1) else: + if X_approx_distribution == XApproxDistribution.COUNT: + X = np.log1p(X) mean = X.mean(axis=0) dfm = X - mean sumsq = np.sum(np.multiply(dfm, dfm), axis=0) v = sumsq / (n - 1) + # AnnData does not guarantee that operations on a view of X will + # return an ndarray, so force the cast if it wasn't done for us. + if type(mean) is not np.ndarray: + mean = mean.toarray() + if type(v) is not np.ndarray: + v = v.toarray() + if fp_err_occurred: mean[np.isfinite(mean) == False] = 0 # noqa: E712 v[np.isfinite(v) == False] = 0 # noqa: E712 diff --git a/backend/common/compute/estimate_distribution.py b/backend/common/compute/estimate_distribution.py new file mode 100644 index 00000000..69fea708 --- /dev/null +++ b/backend/common/compute/estimate_distribution.py @@ -0,0 +1,62 @@ +import numba +import concurrent.futures +import numpy as np +from scipy import sparse +from backend.common.constants import XApproxDistribution + + +@numba.njit(fastmath=True, error_model="numpy", nogil=True) +def min_max(arr): + """Return (min, max) values for the ndarray.""" + n = arr.size + odd = n % 2 + if not odd: + n -= 1 + max_val = min_val = arr[0] + i = 1 + while i < n: + x = arr[i] + y = arr[i + 1] + if x > y: + x, y = y, x + min_val = min(x, min_val) + max_val = max(y, max_val) + i += 2 + if not odd: + x = arr[n] + min_val = min(x, min_val) + max_val = max(x, max_val) + return min_val, max_val + + +def estimate_approximate_distribution(X) -> XApproxDistribution: + """ + Estimate the distribution (normal, count) of the X matrix. + + Currently this is based upon the assumption that scRNA-seq data is + exponentially distributed in its raw (count) form, and when logged, + any (max-min) range in excess of 24 is implies tens of millions of + observations of a single feature and so is extremely unlikely. + """ + if sparse.isspmatrix_csc(X) or sparse.isspmatrix_csr(X): + Xdata = X.data + elif type(X) is np.ndarray: + Xdata = X.reshape( + X.size, + ) + else: + raise TypeError(f"Unsupported matrix type: {str(type(X))}") + + CHUNKSIZE = 1 << 24 + if Xdata.size > CHUNKSIZE: + min_val = max_val = Xdata[0] + with concurrent.futures.ThreadPoolExecutor() as tp: + for (_min, _max) in tp.map(min_max, [Xdata[i : i + CHUNKSIZE] for i in range(0, Xdata.size, CHUNKSIZE)]): + min_val = min(_min, min_val) + max_val = max(_max, max_val) + + else: + min_val, max_val = min_max(Xdata) + + excess_range = (max_val - min_val) > 24 + return XApproxDistribution.COUNT if excess_range else XApproxDistribution.NORMAL diff --git a/backend/common/constants.py b/backend/common/constants.py index a883e23a..4d80a5a0 100644 --- a/backend/common/constants.py +++ b/backend/common/constants.py @@ -24,6 +24,11 @@ class DiffExpMode(AugmentedEnum): VAR_FILTER = "varFilter" +class XApproxDistribution(AugmentedEnum): + NORMAL = "normal" + COUNT = "count" + + JSON_NaN_to_num_warning_msg = "JSON encoding failure - please verify all data are finite values (no NaN or Infinities)" REACTIVE_LIMIT = 1_000_000 diff --git a/backend/czi_hosted/common/config/dataset_config.py b/backend/czi_hosted/common/config/dataset_config.py index 12f46df6..8432c278 100644 --- a/backend/czi_hosted/common/config/dataset_config.py +++ b/backend/czi_hosted/common/config/dataset_config.py @@ -44,6 +44,8 @@ class DatasetConfig(BaseConfig): self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"] self.diffexp__top_n = default_config["diffexp"]["top_n"] + self.X_approx_distribution = default_config["X_approx_distribution"] + except KeyError as e: raise ConfigurationError(f"Unexpected config: {str(e)}") @@ -58,6 +60,7 @@ class DatasetConfig(BaseConfig): self.handle_user_annotations(context) self.handle_embeddings() self.handle_diffexp(context) + self.handle_X_approx_distribution() def handle_app(self): self.validate_correct_type_of_configuration_attribute("app__scripts", list) @@ -199,3 +202,10 @@ class DatasetConfig(BaseConfig): "CAUTION: due to the size of your dataset, " "running differential expression may take longer or fail." ) + + def handle_X_approx_distribution(self): + self.validate_correct_type_of_configuration_attribute("X_approx_distribution", str) + if self.X_approx_distribution not in ["normal", "count"]: + raise ConfigurationError( + "X_approx_distribution has unknown value -- must be 'normal' or 'count'." + ) diff --git a/backend/czi_hosted/data_anndata/anndata_adaptor.py b/backend/czi_hosted/data_anndata/anndata_adaptor.py index c1968b08..f6d3447a 100644 --- a/backend/czi_hosted/data_anndata/anndata_adaptor.py +++ b/backend/czi_hosted/data_anndata/anndata_adaptor.py @@ -8,9 +8,9 @@ from scipy import sparse import backend.common.compute.diffexp_generic as diffexp_generic from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors -from backend.common.constants import Axis, MAX_LAYOUTS +from backend.common.constants import Axis, MAX_LAYOUTS, XApproxDistribution from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata -from backend.common.errors import PrepareError, DatasetAccessError +from backend.common.errors import PrepareError, DatasetAccessError, ConfigurationError from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array from backend.czi_hosted.data_common.data_adaptor import DataAdaptor from backend.common.fbs.matrix import encode_matrix_fbs @@ -28,6 +28,7 @@ class AnndataAdaptor(DataAdaptor): def __init__(self, data_locator, app_config=None, dataset_config=None): super().__init__(data_locator, app_config, dataset_config) self.data = None + self.X_approx_distribution = None self._load_data(data_locator) self._validate_and_initialize() @@ -190,6 +191,10 @@ class AnndataAdaptor(DataAdaptor): self.gene_count = self.data.shape[1] self._create_schema() + if self.dataset_config.X_approx_distribution == "auto": + raise ConfigurationError("X-approx-distribution 'auto' mode unsupported.") + self.X_approx_distribution = self.dataset_config.X_approx_distribution + # heuristic n_values = self.data.shape[0] * self.data.shape[1] if (n_values > 1e8 and self.server_config.adaptor__anndata_adaptor__backed is True) or (n_values > 5e8): @@ -309,13 +314,22 @@ class AnndataAdaptor(DataAdaptor): return convert_anndata_category_colors_to_cxg_category_colors(self.data) def get_X_array(self, obs_mask=None, var_mask=None): + # H5Py does not support boolean indexing (masks), so convert to integer indexing + # when backed (ie, when AnnData is using H5Py indexing) if obs_mask is None: obs_mask = slice(None) + elif self.data.isbacked and obs_mask.dtype == bool: + obs_mask = obs_mask.nonzero()[0] if var_mask is None: var_mask = slice(None) + elif self.data.isbacked and var_mask.dtype == bool: + var_mask = var_mask.nonzero()[0] X = self.data.X[obs_mask, var_mask] return X + def get_X_approx_distribution(self) -> XApproxDistribution: + return self.X_approx_distribution + def get_shape(self): return self.data.shape diff --git a/backend/czi_hosted/data_common/data_adaptor.py b/backend/czi_hosted/data_common/data_adaptor.py index 7780277f..8c3aadc1 100644 --- a/backend/czi_hosted/data_common/data_adaptor.py +++ b/backend/czi_hosted/data_common/data_adaptor.py @@ -7,8 +7,14 @@ from scipy import sparse from server_timing import Timing as ServerTiming from backend.czi_hosted.common.config.app_config import AppConfig -from backend.common.constants import Axis -from backend.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError, UnsupportedSummaryMethod, DatasetAccessError +from backend.common.constants import Axis, XApproxDistribution +from backend.common.errors import ( + FilterError, + JSONEncodingValueError, + ExceedsLimitError, + UnsupportedSummaryMethod, + DatasetAccessError, +) from backend.common.utils.utils import jsonify_numpy from backend.common.fbs.matrix import encode_matrix_fbs @@ -77,6 +83,11 @@ class DataAdaptor(metaclass=ABCMeta): the return type is either ndarray or scipy.sparse.spmatrix.""" pass + @abstractmethod + def get_X_approx_distribution(self) -> XApproxDistribution: + """return the approximate distribution of the X matrix.""" + pass + @abstractmethod def get_shape(self): pass @@ -158,7 +169,7 @@ class DataAdaptor(metaclass=ABCMeta): mask = np.zeros((count,), dtype=np.bool) for i in filter: if type(i) == list: - mask[i[0]: i[1]] = True + mask[i[0] : i[1]] = True else: mask[i] = True return mask @@ -316,12 +327,13 @@ class DataAdaptor(metaclass=ABCMeta): top_n = self.dataset_config.diffexp__top_n if self.server_config.exceeds_limit( - "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B) + "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B) ): raise ExceedsLimitError("Diffexp request exceeds max cell count limit") result = self.compute_diffexp_ttest( - maskA=obs_mask_A, maskB=obs_mask_B, top_n=top_n, lfc_cutoff=self.dataset_config.diffexp__lfc_cutoff) + maskA=obs_mask_A, maskB=obs_mask_B, top_n=top_n, lfc_cutoff=self.dataset_config.diffexp__lfc_cutoff + ) try: return jsonify_numpy(result) diff --git a/backend/czi_hosted/data_cxg/cxg_adaptor.py b/backend/czi_hosted/data_cxg/cxg_adaptor.py index 3cbb0076..4afeafcd 100644 --- a/backend/czi_hosted/data_cxg/cxg_adaptor.py +++ b/backend/czi_hosted/data_cxg/cxg_adaptor.py @@ -8,7 +8,7 @@ import pandas as pd import tiledb from server_timing import Timing as ServerTiming -from backend.common.constants import Axis +from backend.common.constants import Axis, XApproxDistribution from backend.common.errors import DatasetAccessError, ConfigurationError from backend.czi_hosted.common.immutable_kvcache import ImmutableKVCache from backend.common.utils.type_conversion_utils import get_schema_type_hint_from_dtype @@ -37,6 +37,7 @@ class CxgAdaptor(DataAdaptor): self.lsuri_results = ImmutableKVCache(lambda key: self._lsuri(uri=key, tiledb_ctx=self.tiledb_ctx)) self.arrays = ImmutableKVCache(lambda key: self._open_array(uri=key, tiledb_ctx=self.tiledb_ctx)) self.schema = None + self.X_approx_distribution = None self._validate_and_initialize() @@ -175,6 +176,10 @@ class CxgAdaptor(DataAdaptor): if cxg_version not in ["0.0", "0.1", "0.2.0"]: raise DatasetAccessError(f"cxg matrix is not valid: {self.url}") + if self.dataset_config.X_approx_distribution == "auto": + raise ConfigurationError("X-approx-distribution 'auto' mode unsupported.") + self.X_approx_distribution = self.dataset_config.X_approx_distribution + self.title = title self.about = about self.cxg_version = cxg_version @@ -281,6 +286,9 @@ class CxgAdaptor(DataAdaptor): data = X.multi_index[obs_items, var_items][""] return data + def get_X_approx_distribution(self) -> XApproxDistribution: + return self.X_approx_distribution + def get_shape(self): X = self.open_array("X") return X.shape diff --git a/backend/czi_hosted/default_config.py b/backend/czi_hosted/default_config.py index c2375ef6..436aa739 100644 --- a/backend/czi_hosted/default_config.py +++ b/backend/czi_hosted/default_config.py @@ -203,6 +203,8 @@ dataset: lfc_cutoff: 0.01 top_n: 10 + X_approx_distribution: normal # currently fixed config + external: # You can retrieve configuration parameters from this config file, the environment, # the AWS secrets manager, or from the "cellxgene launch" command line arguments. diff --git a/backend/czi_hosted/requirements-prepare.txt b/backend/czi_hosted/requirements-prepare.txt index df3db83c..ae6d4ca6 100644 --- a/backend/czi_hosted/requirements-prepare.txt +++ b/backend/czi_hosted/requirements-prepare.txt @@ -1,4 +1,4 @@ python-igraph louvain>=0.6 -scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here +scanpy umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn diff --git a/backend/czi_hosted/requirements.txt b/backend/czi_hosted/requirements.txt index 67574b5f..08136006 100644 --- a/backend/czi_hosted/requirements.txt +++ b/backend/czi_hosted/requirements.txt @@ -1,4 +1,4 @@ -anndata>=0.7.0 +anndata>=0.7.6 # we use to_memory(), added in 0.7.6 boto3>=1.12.18 click>=7.1.2 Flask>=1.0.2,<2.0.0 # Flask 2.0 is not compatible with the latest version of Flask-RESTful (0.3.8) @@ -11,7 +11,7 @@ flatbuffers>=1.11.0,<2.0.0 # cellxgene is not compatible with 2.0.0. Requires mi flatten-dict>=0.2.0 fsspec>=0.4.4,<0.8.0 gunicorn>=20.0.4 -h5py<3.0.0 # h5py>=3.0.0 had a breaking change; there is a fix in anndata>=0.7.5 +h5py>=3.0.0 numba>=0.49.1,<0.53.0 numpy>=1.15.0 packaging>=20.0 diff --git a/backend/server/cli/launch.py b/backend/server/cli/launch.py index 19c65942..9a4fad7f 100644 --- a/backend/server/cli/launch.py +++ b/backend/server/cli/launch.py @@ -150,6 +150,14 @@ def dataset_args(func): metavar="", help="URL providing more information about the dataset (hint: must be a fully specified absolute URL).", ) + @click.option( + "--X-approx-distribution", + default=DEFAULT_CONFIG.dataset_config.X_approx_distribution, + show_default=True, + type=click.Choice(["auto", "normal", "count"], case_sensitive=False), + help="Specify the approximate distribution of X matrix values. 'auto' will use a heuristic " + "to determine the approximate distribution. Mode 'auto' is incompatible with --backed.", + ) @functools.wraps(func) def wrapper(*args, **kwargs): return func(*args, **kwargs) @@ -318,6 +326,7 @@ def launch( disable_diffexp, config_file, dump_default_config, + x_approx_distribution, ): """Launch the cellxgene data viewer. This web app lets you explore single-cell expression data. @@ -376,6 +385,7 @@ def launch( embeddings__names=embedding, diffexp__enable=not disable_diffexp, diffexp__lfc_cutoff=diffexp_lfc_cutoff, + X_approx_distribution=x_approx_distribution, ) diff = cli_config.server_config.changes_from_default() diff --git a/backend/server/common/config/dataset_config.py b/backend/server/common/config/dataset_config.py index c99d0b0e..14115af7 100644 --- a/backend/server/common/config/dataset_config.py +++ b/backend/server/common/config/dataset_config.py @@ -38,6 +38,8 @@ class DatasetConfig(BaseConfig): self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"] self.diffexp__top_n = default_config["diffexp"]["top_n"] + self.X_approx_distribution = default_config["X_approx_distribution"] + except KeyError as e: raise ConfigurationError(f"Unexpected config: {str(e)}") @@ -50,6 +52,7 @@ class DatasetConfig(BaseConfig): self.handle_user_annotations(context) self.handle_embeddings() self.handle_diffexp(context) + self.handle_X_approx_distribution() def get_data_adaptor(self): server_config = self.app_config.server_config @@ -182,3 +185,10 @@ class DatasetConfig(BaseConfig): context["messagefn"]( "CAUTION: due to the size of your dataset, " "running differential expression may take longer or fail." ) + + def handle_X_approx_distribution(self): + self.validate_correct_type_of_configuration_attribute("X_approx_distribution", str) + if self.X_approx_distribution not in ["auto", "normal", "count"]: + raise ConfigurationError( + "X_approx_distribution has unknown value -- must be 'auto', 'normal' or 'count'." + ) diff --git a/backend/server/data_anndata/anndata_adaptor.py b/backend/server/data_anndata/anndata_adaptor.py index 51a8de19..69d58fcc 100644 --- a/backend/server/data_anndata/anndata_adaptor.py +++ b/backend/server/data_anndata/anndata_adaptor.py @@ -7,8 +7,9 @@ from pandas.core.dtypes.dtypes import CategoricalDtype from scipy import sparse import backend.common.compute.diffexp_generic as diffexp_generic +import backend.common.compute.estimate_distribution as estimate_distribution from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors -from backend.common.constants import Axis, MAX_LAYOUTS +from backend.common.constants import Axis, MAX_LAYOUTS, XApproxDistribution from backend.server.common.corpora import corpora_get_props_from_anndata from backend.common.errors import PrepareError, DatasetAccessError from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array @@ -28,6 +29,7 @@ class AnndataAdaptor(DataAdaptor): def __init__(self, data_locator, app_config=None, dataset_config=None): super().__init__(data_locator, app_config, dataset_config) self.data = None + self.X_approx_distribution = None self._load_data(data_locator) self._validate_and_initialize() @@ -190,6 +192,13 @@ class AnndataAdaptor(DataAdaptor): self.gene_count = self.data.shape[1] self._create_schema() + if self.dataset_config.X_approx_distribution == "auto": + """Lazy evaluate the heuristic if we are backed.""" + if not self.data.isbacked: + self.X_approx_distribution = estimate_distribution.estimate_approximate_distribution(self.data.X) + else: + self.X_approx_distribution = self.dataset_config.X_approx_distribution + # heuristic n_values = self.data.shape[0] * self.data.shape[1] if (n_values > 1e8 and self.server_config.adaptor__anndata_adaptor__backed is True) or (n_values > 5e8): @@ -309,13 +318,29 @@ class AnndataAdaptor(DataAdaptor): return convert_anndata_category_colors_to_cxg_category_colors(self.data) def get_X_array(self, obs_mask=None, var_mask=None): + # H5Py does not support boolean indexing (masks), so convert to integer indexing + # when backed (ie, when AnnData is using H5Py indexing) if obs_mask is None: obs_mask = slice(None) + elif self.data.isbacked and obs_mask.dtype == bool: + obs_mask = obs_mask.nonzero()[0] if var_mask is None: var_mask = slice(None) + elif self.data.isbacked and var_mask.dtype == bool: + var_mask = var_mask.nonzero()[0] X = self.data.X[obs_mask, var_mask] return X + def get_X_approx_distribution(self) -> XApproxDistribution: + """return the approximate distribution of the X matrix.""" + if self.X_approx_distribution is None: + """Not yet evaluated.""" + assert(self.dataset_config.X_approx_distribution == "auto") + self.data = self.data.to_memory() # loads data + self.X_approx_distribution = estimate_distribution.estimate_approximate_distribution(self.data.X) + + return self.X_approx_distribution + def get_shape(self): return self.data.shape diff --git a/backend/server/data_common/data_adaptor.py b/backend/server/data_common/data_adaptor.py index c41f0555..f0a60592 100644 --- a/backend/server/data_common/data_adaptor.py +++ b/backend/server/data_common/data_adaptor.py @@ -6,7 +6,7 @@ from scipy import sparse from server_timing import Timing as ServerTiming from backend.server.common.config.app_config import AppConfig -from backend.common.constants import Axis +from backend.common.constants import Axis, XApproxDistribution from backend.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError, UnsupportedSummaryMethod from backend.common.utils.utils import jsonify_numpy from backend.common.fbs.matrix import encode_matrix_fbs @@ -72,6 +72,10 @@ class DataAdaptor(metaclass=ABCMeta): the return type is either ndarray or scipy.sparse.spmatrix.""" pass + def get_X_approx_distribution(self) -> XApproxDistribution: + """return the approximate distribution of the X matrix.""" + return XApproxDistribution.NORMAL + @abstractmethod def get_shape(self): pass diff --git a/backend/server/default_config.py b/backend/server/default_config.py index 4d401e65..886de6a9 100644 --- a/backend/server/default_config.py +++ b/backend/server/default_config.py @@ -77,6 +77,8 @@ dataset: lfc_cutoff: 0.01 top_n: 10 + X_approx_distribution: auto + external: # You can retrieve configuration parameters from this config file, the environment, # the AWS secrets manager, or from the "cellxgene launch" command line arguments. diff --git a/backend/server/requirements-prepare.txt b/backend/server/requirements-prepare.txt index 3e0fce44..d278c82f 100644 --- a/backend/server/requirements-prepare.txt +++ b/backend/server/requirements-prepare.txt @@ -1,4 +1,4 @@ python-igraph>=0.8 louvain>=0.6 -scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here +scanpy umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn diff --git a/backend/server/requirements.txt b/backend/server/requirements.txt index 621dfd24..827d5c55 100644 --- a/backend/server/requirements.txt +++ b/backend/server/requirements.txt @@ -1,4 +1,4 @@ -anndata>=0.7.0 +anndata>=0.7.6 # we need to_memory(), added in 0.7.6 boto3>=1.12.18 click>=7.1.2 Flask>=1.0.2,<2.0.0 # Flask 2.0 is not compatible with the latest version of Flask-RESTful (0.3.8) @@ -11,9 +11,9 @@ flatbuffers>=1.11.0,<2.0.0 # cellxgene is not compatible with 2.0.0. Requires mi flatten-dict>=0.2.0 fsspec>=0.4.4,<0.8.0 gunicorn>=20.0.4 -h5py<3.0.0 # h5py>=3.0.0 had a breaking change; there is a fix in anndata>=0.7.5 +h5py>=3.0.0 jinja2>=2.11.3 # Flask sub-dependency. Added due to CVE-2020-28493 -numba>=0.51.2,<0.53.0 +numba>=0.51.2 numpy>=1.17.5 packaging>=20.0 pandas>=1.0,!=1.1 # pandas 1.1 breaks tests, https://github.com/pandas-dev/pandas/issues/35446 diff --git a/backend/test/fixtures/czi_hosted_dataset_config_outline.py b/backend/test/fixtures/czi_hosted_dataset_config_outline.py index 0b0ff7c7..44c06341 100644 --- a/backend/test/fixtures/czi_hosted_dataset_config_outline.py +++ b/backend/test/fixtures/czi_hosted_dataset_config_outline.py @@ -30,4 +30,6 @@ dataset: enable: {enable_difexp} lfc_cutoff: {lfc_cutoff} top_n: {top_n} + + X_approx_distribution: {X_approx_distribution} """ diff --git a/backend/test/fixtures/dataset_config_outline.py b/backend/test/fixtures/dataset_config_outline.py index f51ecb8b..27aeda9f 100644 --- a/backend/test/fixtures/dataset_config_outline.py +++ b/backend/test/fixtures/dataset_config_outline.py @@ -27,4 +27,6 @@ dataset: enable: {enable_difexp} lfc_cutoff: {lfc_cutoff} top_n: {top_n} + + X_approx_distribution: {X_approx_distribution} """ diff --git a/backend/test/test_common/unit/utils/test_type_conversion_utils.py b/backend/test/test_common/unit/utils/test_type_conversion_utils.py index 5bf02d78..70fba125 100644 --- a/backend/test/test_common/unit/utils/test_type_conversion_utils.py +++ b/backend/test/test_common/unit/utils/test_type_conversion_utils.py @@ -70,21 +70,21 @@ class TestTypeConversionUtils(unittest.TestCase): self.assertFalse(can_cast) def test__can_cast_to_int32__int64_is_true(self): - array_to_convert = Series(data=["1", "2", "3"], dtype=np.dtype(np.int64)) + array_to_convert = Series(data=[1, 2, 3], dtype=np.dtype(np.int64)) can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert) self.assertTrue(can_cast) def test__can_cast_to_int32__int16_is_true(self): - array_to_convert = Series(data=["1", "2", "3"], dtype=np.dtype(np.int16)) + array_to_convert = Series(data=[1, 2, 3], dtype=np.dtype(np.int16)) can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert) self.assertTrue(can_cast) def test__can_cast_to_int32__int64_with_large_value_is_false(self): - array_to_convert = Series(data=["3000000000", "2", "3"], dtype=np.dtype(np.int64)) + array_to_convert = Series(data=[3000000000, 2, 3], dtype=np.dtype(np.int64)) can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert) diff --git a/backend/test/test_czi_hosted/unit/common/config/__init__.py b/backend/test/test_czi_hosted/unit/common/config/__init__.py index b81d747e..77a9a6d3 100644 --- a/backend/test/test_czi_hosted/unit/common/config/__init__.py +++ b/backend/test/test_czi_hosted/unit/common/config/__init__.py @@ -131,6 +131,7 @@ class ConfigTests(BaseTest): environment=None, aws_secrets_manager_region=None, aws_secrets_manager_secrets=[], + X_approx_distribution="normal", config_file_name="app_config.yml", ): random_num = random.randrange(999999) @@ -194,6 +195,7 @@ class ConfigTests(BaseTest): enable_difexp=enable_difexp, lfc_cutoff=lfc_cutoff, top_n=top_n, + X_approx_distribution=X_approx_distribution, config_file_name=f"temp_dataset_config_{random_num}.yml", ) external_config = self.custom_external_config( @@ -229,6 +231,7 @@ class ConfigTests(BaseTest): enable_difexp="true", lfc_cutoff=0.01, top_n=10, + X_approx_distribution="normal", config_file_name="dataset_config.yml", ): configfile = os.path.join(self.tmp_fixtures_directory, config_file_name) diff --git a/backend/test/test_czi_hosted/unit/common/config/test_dataset_config.py b/backend/test/test_czi_hosted/unit/common/config/test_dataset_config.py index 0437b31b..149e5373 100644 --- a/backend/test/test_czi_hosted/unit/common/config/test_dataset_config.py +++ b/backend/test/test_czi_hosted/unit/common/config/test_dataset_config.py @@ -49,7 +49,7 @@ class TestDatasetConfig(ConfigTests): def test_complete_config_checks_all_attr(self, mock_check_attrs): mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute() self.dataset_config.complete_config(self.context) - self.assertEqual(mock_check_attrs.call_count, 18) + self.assertEqual(mock_check_attrs.call_count, 19) def test_app_sets_script_vars(self): config = self.get_config(scripts=["path/to/script"]) diff --git a/backend/test/test_czi_hosted/unit/compute/test_diffexp_cxg.py b/backend/test/test_czi_hosted/unit/compute/test_diffexp_cxg.py index 8e0e29e2..9134a163 100644 --- a/backend/test/test_czi_hosted/unit/compute/test_diffexp_cxg.py +++ b/backend/test/test_czi_hosted/unit/compute/test_diffexp_cxg.py @@ -20,6 +20,7 @@ class DiffExpTest(unittest.TestCase): adaptor types and different algorithms.""" def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}): + extra_dataset_config["X_approx_distribution"] = "normal" # hardwired for now config = app_config(path, extra_server_config=extra_server_config, extra_dataset_config=extra_dataset_config) loader = MatrixDataLoader(path) adaptor = loader.open(config) @@ -46,28 +47,28 @@ class DiffExpTest(unittest.TestCase): """Checks the results for a specific set of rows selections""" positive_expects = [ - [1712, -0.5525154, 0.0051788902660723345, 1.0], - [1575, 1.0317602, 0.007830310753043345, 1.0], - [693, 0.4703904, 0.008715846769131548, 1.0], - [916, 0.9567287, 0.009080596532247588, 1.0], - [77, 0.02665649, 0.010070392939027756, 1.0], - [782, -1.0981874, 0.010161745218916036, 1.0], - [913, 0.5683986, 0.010782030711612685, 1.0], - [910, 0.83164597, 0.014596411069229197, 1.0], - [1727, 0.4127781, 0.015168372104237176, 1.0], - [1443, -0.8241895, 0.015337080567465522, 1.0] + [1712, 0.24104056, 0.0051788902660723345, 1.0], + [1575, 0.2615018, 0.007830310753043345, 1.0], + [693, 0.23106655, 0.008715846769131548, 1.0], + [916, 0.2395215, 0.009080596532247588, 1.0], + [77, 0.22927025, 0.010070392939027756, 1.0], + [782, 0.20581803, 0.010161745218916036, 1.0], + [913, 0.23841085, 0.010782030711612685, 1.0], + [910, 0.21493295, 0.014596411069229197, 1.0], + [1727, 0.21911663, 0.015168372104237176, 1.0], + [1443, 0.19814226, 0.015337080567465522, 1.0], ] negative_expects = [ - [956, 0.016060986, 0.0008649321884808977, 1.0], - [1124, 0.96602094, 0.0011717216548271284, 1.0], - [1809, 1.1110606, 0.0019304405196777848, 1.0], - [1754, 0.5201581, 0.005691734062127954, 1.0], - [948, 1.6390722, 0.006622111055981219, 1.0], - [1810, 0.78618884, 0.007055917428377063, 1.0], - [779, 1.5241305, 0.007202934422407284, 1.0], - [576, 0.97873515, 0.008272092578813124, 1.0], - [538, 0.89114505, 0.01062259019889307, 1.0], - [436, 0.3119122, 0.01127515110543434, 1.0] + [956, -0.29662406, 0.0008649321884808977, 1.0], + [1124, -0.2607333, 0.0011717216548271284, 1.0], + [1809, -0.24854594, 0.0019304405196777848, 1.0], + [1754, -0.24683577, 0.005691734062127954, 1.0], + [948, -0.18708363, 0.006622111055981219, 1.0], + [1810, -0.2172082, 0.007055917428377063, 1.0], + [779, -0.21150622, 0.007202934422407284, 1.0], + [576, -0.19008157, 0.008272092578813124, 1.0], + [538, -0.21803819, 0.01062259019889307, 1.0], + [436, -0.2100364, 0.01127515110543434, 1.0], ] self.compare_diffexp_results(results['positive'], positive_expects) diff --git a/backend/test/test_server/unit/common/config/__init__.py b/backend/test/test_server/unit/common/config/__init__.py index d954e3ff..6f0c48eb 100644 --- a/backend/test/test_server/unit/common/config/__init__.py +++ b/backend/test/test_server/unit/common/config/__init__.py @@ -103,6 +103,7 @@ class ConfigTests(unittest.TestCase): environment=None, aws_secrets_manager_region=None, aws_secrets_manager_secrets=[], + X_approx_distribution="auto", config_file_name="app_config.yml", ): random_num = random.randrange(999999) @@ -150,6 +151,7 @@ class ConfigTests(unittest.TestCase): enable_difexp=enable_difexp, lfc_cutoff=lfc_cutoff, top_n=top_n, + X_approx_distribution=X_approx_distribution, config_file_name=f"temp_dataset_config_{random_num}.yml", ) external_config = self.custom_external_config( @@ -185,6 +187,7 @@ class ConfigTests(unittest.TestCase): enable_difexp="true", lfc_cutoff=0.01, top_n=10, + X_approx_distribution="auto", config_file_name="dataset_config.yml", ): configfile = os.path.join(self.tmp_fixtures_directory, config_file_name) diff --git a/backend/test/test_server/unit/common/config/test_dataset_config.py b/backend/test/test_server/unit/common/config/test_dataset_config.py index 6e55923a..2f923246 100644 --- a/backend/test/test_server/unit/common/config/test_dataset_config.py +++ b/backend/test/test_server/unit/common/config/test_dataset_config.py @@ -7,7 +7,7 @@ from unittest.mock import patch from backend.server.common.annotations.local_file_csv import AnnotationsLocalFile from backend.server.common.config.app_config import AppConfig from backend.server.common.config.base_config import BaseConfig -from backend.test import FIXTURES_ROOT, H5AD_FIXTURE +from backend.test import H5AD_FIXTURE from backend.common.errors import ConfigurationError from backend.test.test_server.unit.common.config import ConfigTests @@ -46,7 +46,7 @@ class TestDatasetConfig(ConfigTests): mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute() self.dataset_config.complete_config(self.context) self.assertIsNotNone(self.config.server_config.data_adaptor) - self.assertEqual(mock_check_attrs.call_count, 16) + self.assertEqual(mock_check_attrs.call_count, 17) def test_app_sets_script_vars(self): config = self.get_config(scripts=["path/to/script"]) diff --git a/backend/test/test_server/unit/compute/test_diffexp_h5ad.py b/backend/test/test_server/unit/compute/test_diffexp_h5ad.py index 3f98a023..739888ce 100644 --- a/backend/test/test_server/unit/compute/test_diffexp_h5ad.py +++ b/backend/test/test_server/unit/compute/test_diffexp_h5ad.py @@ -38,28 +38,28 @@ class DiffExpTest(unittest.TestCase): """Checks the results for a specific set of rows selections""" positive_expects = [ - [1712, -0.5525154, 0.0051788902660723345, 1.0], - [1575, 1.0317602, 0.007830310753043345, 1.0], - [693, 0.4703904, 0.008715846769131548, 1.0], - [916, 0.9567287, 0.009080596532247588, 1.0], - [77, 0.02665649, 0.010070392939027756, 1.0], - [782, -1.0981874, 0.010161745218916036, 1.0], - [913, 0.5683986, 0.010782030711612685, 1.0], - [910, 0.83164597, 0.014596411069229197, 1.0], - [1727, 0.4127781, 0.015168372104237176, 1.0], - [1443, -0.8241895, 0.015337080567465522, 1.0] + [1712, 0.24104056, 0.0051788902660723345, 1.0], + [1575, 0.2615018, 0.007830310753043345, 1.0], + [693, 0.23106655, 0.008715846769131548, 1.0], + [916, 0.2395215, 0.009080596532247588, 1.0], + [77, 0.22927025, 0.010070392939027756, 1.0], + [782, 0.20581803, 0.010161745218916036, 1.0], + [913, 0.23841085, 0.010782030711612685, 1.0], + [910, 0.21493295, 0.014596411069229197, 1.0], + [1727, 0.21911663, 0.015168372104237176, 1.0], + [1443, 0.19814226, 0.015337080567465522, 1.0], ] negative_expects = [ - [956, 0.016060986, 0.0008649321884808977, 1.0], - [1124, 0.96602094, 0.0011717216548271284, 1.0], - [1809, 1.1110606, 0.0019304405196777848, 1.0], - [1754, 0.5201581, 0.005691734062127954, 1.0], - [948, 1.6390722, 0.006622111055981219, 1.0], - [1810, 0.78618884, 0.007055917428377063, 1.0], - [779, 1.5241305, 0.007202934422407284, 1.0], - [576, 0.97873515, 0.008272092578813124, 1.0], - [538, 0.89114505, 0.01062259019889307, 1.0], - [436, 0.3119122, 0.01127515110543434, 1.0] + [956, -0.29662406, 0.0008649321884808977, 1.0], + [1124, -0.2607333, 0.0011717216548271284, 1.0], + [1809, -0.24854594, 0.0019304405196777848, 1.0], + [1754, -0.24683577, 0.005691734062127954, 1.0], + [948, -0.18708363, 0.006622111055981219, 1.0], + [1810, -0.2172082, 0.007055917428377063, 1.0], + [779, -0.21150622, 0.007202934422407284, 1.0], + [576, -0.19008157, 0.008272092578813124, 1.0], + [538, -0.21803819, 0.01062259019889307, 1.0], + [436, -0.2100364, 0.01127515110543434, 1.0], ] self.compare_diffexp_results(results["positive"], positive_expects) diff --git a/backend/test/test_server/unit/compute/test_est_dist.py b/backend/test/test_server/unit/compute/test_est_dist.py new file mode 100644 index 00000000..85cb88de --- /dev/null +++ b/backend/test/test_server/unit/compute/test_est_dist.py @@ -0,0 +1,41 @@ +import unittest +import numpy as np +from scipy import sparse +from backend.common.compute.estimate_distribution import estimate_approximate_distribution +from backend.common.constants import XApproxDistribution +from backend.server.data_common.matrix_loader import MatrixDataLoader +from backend.test.test_server.unit import app_config +from backend.test import PROJECT_ROOT + + +class EstDistTest(unittest.TestCase): + """Tests the diffexp returns the expected results for one test case, using the h5ad + adaptor types and different algorithms.""" + + def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}): + config = app_config(path, extra_server_config=extra_server_config, extra_dataset_config=extra_dataset_config) + loader = MatrixDataLoader(path) + adaptor = loader.open(config) + return adaptor + + def test_adaptestimate_approximate_distribution(self): + adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad") + self.assertEqual(adaptor.get_X_approx_distribution(), XApproxDistribution.NORMAL) + + def test_estimate_approximate_distribution(self): + raw = np.random.exponential(scale=1000, size=(100, 40)) + + # ndarray + self.assertEqual(estimate_approximate_distribution(raw), XApproxDistribution.COUNT) + self.assertEqual(estimate_approximate_distribution(np.log1p(raw)), XApproxDistribution.NORMAL) + + # csr_matrix + self.assertEqual(estimate_approximate_distribution(sparse.csr_matrix(raw)), XApproxDistribution.COUNT) + self.assertEqual( + estimate_approximate_distribution(sparse.csr_matrix(np.log1p(raw))), XApproxDistribution.NORMAL + ) + + # BIG (ie, trigger MT) + big = np.random.exponential(scale=100, size=(1_000_000, 100)) + self.assertEqual(estimate_approximate_distribution(big), XApproxDistribution.COUNT) + self.assertEqual(estimate_approximate_distribution(np.log1p(big)), XApproxDistribution.NORMAL) diff --git a/backend/test/test_server/unit/data_anndata/test_anndata_adaptor.py b/backend/test/test_server/unit/data_anndata/test_anndata_adaptor.py index 3865eeaa..2d046bde 100644 --- a/backend/test/test_server/unit/data_anndata/test_anndata_adaptor.py +++ b/backend/test/test_server/unit/data_anndata/test_anndata_adaptor.py @@ -22,19 +22,27 @@ Test the anndata adaptor using the pbmc3k data set. @parameterized_class( - ("data_locator", "backed"), + ("data_locator", "backed", "X_approx_distribution"), [ - (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False), - (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", False), - (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", False), - (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", True), - (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", True), - (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", True), + (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False, "auto"), + (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", False, "auto"), + (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", False, "auto"), + (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", True, "auto"), + (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", True, "auto"), + (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", True, "auto"), + (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False, "normal"), + (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", False, "normal"), + (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", False, "normal"), + (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", True, "normal"), + (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", True, "normal"), + (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", True, "normal"), ], ) class AdaptorTest(unittest.TestCase): def setUp(self): - config = app_config(self.data_locator, self.backed) + config = app_config( + self.data_locator, self.backed, extra_dataset_config=dict(X_approx_distribution=self.X_approx_distribution) + ) self.data = AnndataAdaptor(DataLocator(self.data_locator), config) def test_init(self): @@ -90,7 +98,8 @@ class AdaptorTest(unittest.TestCase): def test_schema_produces_error(self): self.data.data.obs["time"] = pd.Series( - list([time.time() for i in range(self.data.cell_count)]), dtype="datetime64[ns]", + list([time.time() for i in range(self.data.cell_count)]), + dtype="datetime64[ns]", ) with pytest.raises(TypeError): self.data._create_schema() @@ -107,7 +116,7 @@ class AdaptorTest(unittest.TestCase): self.assertTrue((Y >= 0).all() and (Y <= 1).all()) def test_layout_fields(self): - """ X_pca, X_tsne, X_umap are available """ + """X_pca, X_tsne, X_umap are available""" fbs = self.data.layout_to_fbs_matrix(["pca"]) layout = decode_fbs.decode_matrix_FBS(fbs) self.assertEqual(layout["n_cols"], 2) @@ -127,7 +136,8 @@ class AdaptorTest(unittest.TestCase): self.assertEqual(annotations["n_cols"], 5) obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"] self.assertEqual( - annotations["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"], + annotations["col_idx"], + [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"], ) fbs = self.data.annotation_to_fbs_matrix("var") @@ -153,12 +163,12 @@ class AdaptorTest(unittest.TestCase): f1 = {"filter": {"obs": {"index": [[0, 500]]}}} f2 = {"filter": {"obs": {"index": [[500, 1000]]}}} result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"])) - self.assertEqual(len(result['positive']), 10) - self.assertEqual(len(result['negative']), 10) + self.assertEqual(len(result["positive"]), 10) + self.assertEqual(len(result["negative"]), 10) result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20)) - self.assertEqual(len(result['positive']), 20) - self.assertEqual(len(result['negative']), 20) + self.assertEqual(len(result["positive"]), 20) + self.assertEqual(len(result["negative"]), 20) def test_data_frame(self): f1 = {"var": {"index": [[0, 10]]}}