rename X_approx_distribution to X_approximate_distribution (#2337)

This commit is contained in:
Bruce Martin
2021-07-27 13:43:04 -07:00
committed by GitHub
parent 1998c0ad63
commit 0b1ab02a60
20 changed files with 79 additions and 77 deletions
+7 -7
View File
@@ -1,6 +1,6 @@
import numpy as np import numpy as np
from scipy import sparse, stats from scipy import sparse, stats
from backend.common.constants import XApproxDistribution from backend.common.constants import XApproximateDistribution
def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01): def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
@@ -30,13 +30,13 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
:return: for top N genes, {"positive": for top N genes, [ varindex, foldchange, pval, pval_adj ], "negative": for top N genes, [ varindex, foldchange, pval, pval_adj ]} :return: for top N genes, {"positive": for top N genes, [ varindex, foldchange, pval, pval_adj ], "negative": for top N genes, [ varindex, foldchange, pval, pval_adj ]}
""" """
X_approx_distribution = adaptor.get_X_approx_distribution() X_approximate_distribution = adaptor.get_X_approximate_distribution()
dataA = adaptor.get_X_array(maskA, None) dataA = adaptor.get_X_array(maskA, None)
dataB = adaptor.get_X_array(maskB, None) dataB = adaptor.get_X_array(maskB, None)
# mean, variance, N - calculate for both selections # mean, variance, N - calculate for both selections
meanA, vA, nA = mean_var_n(dataA, X_approx_distribution) meanA, vA, nA = mean_var_n(dataA, X_approximate_distribution)
meanB, vB, nB = mean_var_n(dataB, X_approx_distribution) meanB, vB, nB = mean_var_n(dataB, X_approximate_distribution)
res = diffexp_ttest_from_mean_var(meanA, vA, nA, meanB, vB, nB, top_n, diffexp_lfc_cutoff) res = diffexp_ttest_from_mean_var(meanA, vA, nA, meanB, vB, nB, top_n, diffexp_lfc_cutoff)
return res return res
@@ -113,7 +113,7 @@ def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp
# Convenience function which handles sparse data # Convenience function which handles sparse data
def mean_var_n(X, X_approx_distribution=XApproxDistribution.NORMAL): def mean_var_n(X, X_approximate_distribution=XApproximateDistribution.NORMAL):
""" """
Two-pass variance calculation. Numerically (more) stable Two-pass variance calculation. Numerically (more) stable
than naive methods (and same method used by numpy.var()) than naive methods (and same method used by numpy.var())
@@ -131,14 +131,14 @@ def mean_var_n(X, X_approx_distribution=XApproxDistribution.NORMAL):
with np.errstate(divide="call", invalid="call", call=fp_err_set): with np.errstate(divide="call", invalid="call", call=fp_err_set):
n = X.shape[0] n = X.shape[0]
if sparse.issparse(X): if sparse.issparse(X):
if X_approx_distribution == XApproxDistribution.COUNT: if X_approximate_distribution == XApproximateDistribution.COUNT:
X = X.log1p() X = X.log1p()
mean = X.mean(axis=0).A1 mean = X.mean(axis=0).A1
dfm = X - mean dfm = X - mean
sumsq = np.sum(np.multiply(dfm, dfm), axis=0).A1 sumsq = np.sum(np.multiply(dfm, dfm), axis=0).A1
v = sumsq / (n - 1) v = sumsq / (n - 1)
else: else:
if X_approx_distribution == XApproxDistribution.COUNT: if X_approximate_distribution == XApproximateDistribution.COUNT:
X = np.log1p(X) X = np.log1p(X)
mean = X.mean(axis=0) mean = X.mean(axis=0)
dfm = X - mean dfm = X - mean
@@ -2,7 +2,7 @@ import numba
import concurrent.futures import concurrent.futures
import numpy as np import numpy as np
from scipy import sparse from scipy import sparse
from backend.common.constants import XApproxDistribution from backend.common.constants import XApproximateDistribution
@numba.njit(fastmath=True, error_model="numpy", nogil=True) @numba.njit(fastmath=True, error_model="numpy", nogil=True)
@@ -29,7 +29,7 @@ def min_max(arr):
return min_val, max_val return min_val, max_val
def estimate_approximate_distribution(X) -> XApproxDistribution: def estimate_approximate_distribution(X) -> XApproximateDistribution:
""" """
Estimate the distribution (normal, count) of the X matrix. Estimate the distribution (normal, count) of the X matrix.
@@ -59,4 +59,4 @@ def estimate_approximate_distribution(X) -> XApproxDistribution:
min_val, max_val = min_max(Xdata) min_val, max_val = min_max(Xdata)
excess_range = (max_val - min_val) > 24 excess_range = (max_val - min_val) > 24
return XApproxDistribution.COUNT if excess_range else XApproxDistribution.NORMAL return XApproximateDistribution.COUNT if excess_range else XApproximateDistribution.NORMAL
+1 -1
View File
@@ -24,7 +24,7 @@ class DiffExpMode(AugmentedEnum):
VAR_FILTER = "varFilter" VAR_FILTER = "varFilter"
class XApproxDistribution(AugmentedEnum): class XApproximateDistribution(AugmentedEnum):
NORMAL = "normal" NORMAL = "normal"
COUNT = "count" COUNT = "count"
@@ -44,7 +44,7 @@ class DatasetConfig(BaseConfig):
self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"] self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
self.diffexp__top_n = default_config["diffexp"]["top_n"] self.diffexp__top_n = default_config["diffexp"]["top_n"]
self.X_approx_distribution = default_config["X_approx_distribution"] self.X_approximate_distribution = default_config["X_approximate_distribution"]
except KeyError as e: except KeyError as e:
raise ConfigurationError(f"Unexpected config: {str(e)}") raise ConfigurationError(f"Unexpected config: {str(e)}")
@@ -60,7 +60,7 @@ class DatasetConfig(BaseConfig):
self.handle_user_annotations(context) self.handle_user_annotations(context)
self.handle_embeddings() self.handle_embeddings()
self.handle_diffexp(context) self.handle_diffexp(context)
self.handle_X_approx_distribution() self.handle_X_approximate_distribution()
def handle_app(self): def handle_app(self):
self.validate_correct_type_of_configuration_attribute("app__scripts", list) self.validate_correct_type_of_configuration_attribute("app__scripts", list)
@@ -203,9 +203,9 @@ class DatasetConfig(BaseConfig):
"running differential expression may take longer or fail." "running differential expression may take longer or fail."
) )
def handle_X_approx_distribution(self): def handle_X_approximate_distribution(self):
self.validate_correct_type_of_configuration_attribute("X_approx_distribution", str) self.validate_correct_type_of_configuration_attribute("X_approximate_distribution", str)
if self.X_approx_distribution not in ["normal", "count"]: if self.X_approximate_distribution not in ["normal", "count"]:
raise ConfigurationError( raise ConfigurationError(
"X_approx_distribution has unknown value -- must be 'normal' or 'count'." "X_approximate_distribution has unknown value -- must be 'normal' or 'count'."
) )
@@ -8,7 +8,7 @@ from scipy import sparse
import backend.common.compute.diffexp_generic as diffexp_generic import backend.common.compute.diffexp_generic as diffexp_generic
from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
from backend.common.constants import Axis, MAX_LAYOUTS, XApproxDistribution from backend.common.constants import Axis, MAX_LAYOUTS, XApproximateDistribution
from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata
from backend.common.errors import PrepareError, DatasetAccessError, ConfigurationError from backend.common.errors import PrepareError, DatasetAccessError, ConfigurationError
from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
@@ -28,7 +28,7 @@ class AnndataAdaptor(DataAdaptor):
def __init__(self, data_locator, app_config=None, dataset_config=None): def __init__(self, data_locator, app_config=None, dataset_config=None):
super().__init__(data_locator, app_config, dataset_config) super().__init__(data_locator, app_config, dataset_config)
self.data = None self.data = None
self.X_approx_distribution = None self.X_approximate_distribution = None
self._load_data(data_locator) self._load_data(data_locator)
self._validate_and_initialize() self._validate_and_initialize()
@@ -191,9 +191,9 @@ class AnndataAdaptor(DataAdaptor):
self.gene_count = self.data.shape[1] self.gene_count = self.data.shape[1]
self._create_schema() self._create_schema()
if self.dataset_config.X_approx_distribution == "auto": if self.dataset_config.X_approximate_distribution == "auto":
raise ConfigurationError("X-approx-distribution 'auto' mode unsupported.") raise ConfigurationError("X-approximate-distribution 'auto' mode unsupported.")
self.X_approx_distribution = self.dataset_config.X_approx_distribution self.X_approximate_distribution = self.dataset_config.X_approximate_distribution
# heuristic # heuristic
n_values = self.data.shape[0] * self.data.shape[1] n_values = self.data.shape[0] * self.data.shape[1]
@@ -327,8 +327,8 @@ class AnndataAdaptor(DataAdaptor):
X = self.data.X[obs_mask, var_mask] X = self.data.X[obs_mask, var_mask]
return X return X
def get_X_approx_distribution(self) -> XApproxDistribution: def get_X_approximate_distribution(self) -> XApproximateDistribution:
return self.X_approx_distribution return self.X_approximate_distribution
def get_shape(self): def get_shape(self):
return self.data.shape return self.data.shape
@@ -7,7 +7,7 @@ from scipy import sparse
from server_timing import Timing as ServerTiming from server_timing import Timing as ServerTiming
from backend.czi_hosted.common.config.app_config import AppConfig from backend.czi_hosted.common.config.app_config import AppConfig
from backend.common.constants import Axis, XApproxDistribution from backend.common.constants import Axis, XApproximateDistribution
from backend.common.errors import ( from backend.common.errors import (
FilterError, FilterError,
JSONEncodingValueError, JSONEncodingValueError,
@@ -84,7 +84,7 @@ class DataAdaptor(metaclass=ABCMeta):
pass pass
@abstractmethod @abstractmethod
def get_X_approx_distribution(self) -> XApproxDistribution: def get_X_approximate_distribution(self) -> XApproximateDistribution:
"""return the approximate distribution of the X matrix.""" """return the approximate distribution of the X matrix."""
pass pass
+7 -7
View File
@@ -8,7 +8,7 @@ import pandas as pd
import tiledb import tiledb
from server_timing import Timing as ServerTiming from server_timing import Timing as ServerTiming
from backend.common.constants import Axis, XApproxDistribution from backend.common.constants import Axis, XApproximateDistribution
from backend.common.errors import DatasetAccessError, ConfigurationError from backend.common.errors import DatasetAccessError, ConfigurationError
from backend.czi_hosted.common.immutable_kvcache import ImmutableKVCache from backend.czi_hosted.common.immutable_kvcache import ImmutableKVCache
from backend.common.utils.type_conversion_utils import get_schema_type_hint_from_dtype from backend.common.utils.type_conversion_utils import get_schema_type_hint_from_dtype
@@ -37,7 +37,7 @@ class CxgAdaptor(DataAdaptor):
self.lsuri_results = ImmutableKVCache(lambda key: self._lsuri(uri=key, tiledb_ctx=self.tiledb_ctx)) self.lsuri_results = ImmutableKVCache(lambda key: self._lsuri(uri=key, tiledb_ctx=self.tiledb_ctx))
self.arrays = ImmutableKVCache(lambda key: self._open_array(uri=key, tiledb_ctx=self.tiledb_ctx)) self.arrays = ImmutableKVCache(lambda key: self._open_array(uri=key, tiledb_ctx=self.tiledb_ctx))
self.schema = None self.schema = None
self.X_approx_distribution = None self.X_approximate_distribution = None
self._validate_and_initialize() self._validate_and_initialize()
@@ -176,9 +176,9 @@ class CxgAdaptor(DataAdaptor):
if cxg_version not in ["0.0", "0.1", "0.2.0"]: if cxg_version not in ["0.0", "0.1", "0.2.0"]:
raise DatasetAccessError(f"cxg matrix is not valid: {self.url}") raise DatasetAccessError(f"cxg matrix is not valid: {self.url}")
if self.dataset_config.X_approx_distribution == "auto": if self.dataset_config.X_approximate_distribution == "auto":
raise ConfigurationError("X-approx-distribution 'auto' mode unsupported.") raise ConfigurationError("X-approximate-distribution 'auto' mode unsupported.")
self.X_approx_distribution = self.dataset_config.X_approx_distribution self.X_approximate_distribution = self.dataset_config.X_approximate_distribution
self.title = title self.title = title
self.about = about self.about = about
@@ -286,8 +286,8 @@ class CxgAdaptor(DataAdaptor):
data = X.multi_index[obs_items, var_items][""] data = X.multi_index[obs_items, var_items][""]
return data return data
def get_X_approx_distribution(self) -> XApproxDistribution: def get_X_approximate_distribution(self) -> XApproximateDistribution:
return self.X_approx_distribution return self.X_approximate_distribution
def get_shape(self): def get_shape(self):
X = self.open_array("X") X = self.open_array("X")
+1 -1
View File
@@ -203,7 +203,7 @@ dataset:
lfc_cutoff: 0.01 lfc_cutoff: 0.01
top_n: 10 top_n: 10
X_approx_distribution: normal # currently fixed config X_approximate_distribution: normal # currently fixed config
external: external:
# You can retrieve configuration parameters from this config file, the environment, # You can retrieve configuration parameters from this config file, the environment,
+4 -4
View File
@@ -151,8 +151,8 @@ def dataset_args(func):
help="URL providing more information about the dataset (hint: must be a fully specified absolute URL).", help="URL providing more information about the dataset (hint: must be a fully specified absolute URL).",
) )
@click.option( @click.option(
"--X-approx-distribution", "--X-approximate-distribution",
default=DEFAULT_CONFIG.dataset_config.X_approx_distribution, default=DEFAULT_CONFIG.dataset_config.X_approximate_distribution,
show_default=True, show_default=True,
type=click.Choice(["auto", "normal", "count"], case_sensitive=False), type=click.Choice(["auto", "normal", "count"], case_sensitive=False),
help="Specify the approximate distribution of X matrix values. 'auto' will use a heuristic " help="Specify the approximate distribution of X matrix values. 'auto' will use a heuristic "
@@ -326,7 +326,7 @@ def launch(
disable_diffexp, disable_diffexp,
config_file, config_file,
dump_default_config, dump_default_config,
x_approx_distribution, x_approximate_distribution,
): ):
"""Launch the cellxgene data viewer. """Launch the cellxgene data viewer.
This web app lets you explore single-cell expression data. This web app lets you explore single-cell expression data.
@@ -385,7 +385,7 @@ def launch(
embeddings__names=embedding, embeddings__names=embedding,
diffexp__enable=not disable_diffexp, diffexp__enable=not disable_diffexp,
diffexp__lfc_cutoff=diffexp_lfc_cutoff, diffexp__lfc_cutoff=diffexp_lfc_cutoff,
X_approx_distribution=x_approx_distribution, X_approximate_distribution=x_approximate_distribution,
) )
diff = cli_config.server_config.changes_from_default() diff = cli_config.server_config.changes_from_default()
@@ -38,7 +38,7 @@ class DatasetConfig(BaseConfig):
self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"] self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
self.diffexp__top_n = default_config["diffexp"]["top_n"] self.diffexp__top_n = default_config["diffexp"]["top_n"]
self.X_approx_distribution = default_config["X_approx_distribution"] self.X_approximate_distribution = default_config["X_approximate_distribution"]
except KeyError as e: except KeyError as e:
raise ConfigurationError(f"Unexpected config: {str(e)}") raise ConfigurationError(f"Unexpected config: {str(e)}")
@@ -52,7 +52,7 @@ class DatasetConfig(BaseConfig):
self.handle_user_annotations(context) self.handle_user_annotations(context)
self.handle_embeddings() self.handle_embeddings()
self.handle_diffexp(context) self.handle_diffexp(context)
self.handle_X_approx_distribution() self.handle_X_approximate_distribution()
def get_data_adaptor(self): def get_data_adaptor(self):
server_config = self.app_config.server_config server_config = self.app_config.server_config
@@ -186,9 +186,9 @@ class DatasetConfig(BaseConfig):
"CAUTION: due to the size of your dataset, " "running differential expression may take longer or fail." "CAUTION: due to the size of your dataset, " "running differential expression may take longer or fail."
) )
def handle_X_approx_distribution(self): def handle_X_approximate_distribution(self):
self.validate_correct_type_of_configuration_attribute("X_approx_distribution", str) self.validate_correct_type_of_configuration_attribute("X_approximate_distribution", str)
if self.X_approx_distribution not in ["auto", "normal", "count"]: if self.X_approximate_distribution not in ["auto", "normal", "count"]:
raise ConfigurationError( raise ConfigurationError(
"X_approx_distribution has unknown value -- must be 'auto', 'normal' or 'count'." "X_approximate_distribution has unknown value -- must be 'auto', 'normal' or 'count'."
) )
+10 -10
View File
@@ -9,7 +9,7 @@ from scipy import sparse
import backend.common.compute.diffexp_generic as diffexp_generic import backend.common.compute.diffexp_generic as diffexp_generic
import backend.common.compute.estimate_distribution as estimate_distribution import backend.common.compute.estimate_distribution as estimate_distribution
from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
from backend.common.constants import Axis, MAX_LAYOUTS, XApproxDistribution from backend.common.constants import Axis, MAX_LAYOUTS, XApproximateDistribution
from backend.server.common.corpora import corpora_get_props_from_anndata from backend.server.common.corpora import corpora_get_props_from_anndata
from backend.common.errors import PrepareError, DatasetAccessError from backend.common.errors import PrepareError, DatasetAccessError
from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
@@ -29,7 +29,7 @@ class AnndataAdaptor(DataAdaptor):
def __init__(self, data_locator, app_config=None, dataset_config=None): def __init__(self, data_locator, app_config=None, dataset_config=None):
super().__init__(data_locator, app_config, dataset_config) super().__init__(data_locator, app_config, dataset_config)
self.data = None self.data = None
self.X_approx_distribution = None self.X_approximate_distribution = None
self._load_data(data_locator) self._load_data(data_locator)
self._validate_and_initialize() self._validate_and_initialize()
@@ -192,12 +192,12 @@ class AnndataAdaptor(DataAdaptor):
self.gene_count = self.data.shape[1] self.gene_count = self.data.shape[1]
self._create_schema() self._create_schema()
if self.dataset_config.X_approx_distribution == "auto": if self.dataset_config.X_approximate_distribution == "auto":
"""Lazy evaluate the heuristic if we are backed.""" """Lazy evaluate the heuristic if we are backed."""
if not self.data.isbacked: if not self.data.isbacked:
self.X_approx_distribution = estimate_distribution.estimate_approximate_distribution(self.data.X) self.X_approximate_distribution = estimate_distribution.estimate_approximate_distribution(self.data.X)
else: else:
self.X_approx_distribution = self.dataset_config.X_approx_distribution self.X_approximate_distribution = self.dataset_config.X_approximate_distribution
# heuristic # heuristic
n_values = self.data.shape[0] * self.data.shape[1] n_values = self.data.shape[0] * self.data.shape[1]
@@ -331,15 +331,15 @@ class AnndataAdaptor(DataAdaptor):
X = self.data.X[obs_mask, var_mask] X = self.data.X[obs_mask, var_mask]
return X return X
def get_X_approx_distribution(self) -> XApproxDistribution: def get_X_approximate_distribution(self) -> XApproximateDistribution:
"""return the approximate distribution of the X matrix.""" """return the approximate distribution of the X matrix."""
if self.X_approx_distribution is None: if self.X_approximate_distribution is None:
"""Not yet evaluated.""" """Not yet evaluated."""
assert(self.dataset_config.X_approx_distribution == "auto") assert(self.dataset_config.X_approximate_distribution == "auto")
self.data = self.data.to_memory() # loads data self.data = self.data.to_memory() # loads data
self.X_approx_distribution = estimate_distribution.estimate_approximate_distribution(self.data.X) self.X_approximate_distribution = estimate_distribution.estimate_approximate_distribution(self.data.X)
return self.X_approx_distribution return self.X_approximate_distribution
def get_shape(self): def get_shape(self):
return self.data.shape return self.data.shape
+3 -3
View File
@@ -6,7 +6,7 @@ from scipy import sparse
from server_timing import Timing as ServerTiming from server_timing import Timing as ServerTiming
from backend.server.common.config.app_config import AppConfig from backend.server.common.config.app_config import AppConfig
from backend.common.constants import Axis, XApproxDistribution from backend.common.constants import Axis, XApproximateDistribution
from backend.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError, UnsupportedSummaryMethod from backend.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError, UnsupportedSummaryMethod
from backend.common.utils.utils import jsonify_numpy from backend.common.utils.utils import jsonify_numpy
from backend.common.fbs.matrix import encode_matrix_fbs from backend.common.fbs.matrix import encode_matrix_fbs
@@ -72,9 +72,9 @@ class DataAdaptor(metaclass=ABCMeta):
the return type is either ndarray or scipy.sparse.spmatrix.""" the return type is either ndarray or scipy.sparse.spmatrix."""
pass pass
def get_X_approx_distribution(self) -> XApproxDistribution: def get_X_approximate_distribution(self) -> XApproximateDistribution:
"""return the approximate distribution of the X matrix.""" """return the approximate distribution of the X matrix."""
return XApproxDistribution.NORMAL return XApproximateDistribution.NORMAL
@abstractmethod @abstractmethod
def get_shape(self): def get_shape(self):
+1 -1
View File
@@ -77,7 +77,7 @@ dataset:
lfc_cutoff: 0.01 lfc_cutoff: 0.01
top_n: 10 top_n: 10
X_approx_distribution: auto X_approximate_distribution: auto
external: external:
# You can retrieve configuration parameters from this config file, the environment, # You can retrieve configuration parameters from this config file, the environment,
+1 -1
View File
@@ -31,5 +31,5 @@ dataset:
lfc_cutoff: {lfc_cutoff} lfc_cutoff: {lfc_cutoff}
top_n: {top_n} top_n: {top_n}
X_approx_distribution: {X_approx_distribution} X_approximate_distribution: {X_approximate_distribution}
""" """
+1 -1
View File
@@ -28,5 +28,5 @@ dataset:
lfc_cutoff: {lfc_cutoff} lfc_cutoff: {lfc_cutoff}
top_n: {top_n} top_n: {top_n}
X_approx_distribution: {X_approx_distribution} X_approximate_distribution: {X_approximate_distribution}
""" """
@@ -131,7 +131,7 @@ class ConfigTests(BaseTest):
environment=None, environment=None,
aws_secrets_manager_region=None, aws_secrets_manager_region=None,
aws_secrets_manager_secrets=[], aws_secrets_manager_secrets=[],
X_approx_distribution="normal", X_approximate_distribution="normal",
config_file_name="app_config.yml", config_file_name="app_config.yml",
): ):
random_num = random.randrange(999999) random_num = random.randrange(999999)
@@ -195,7 +195,7 @@ class ConfigTests(BaseTest):
enable_difexp=enable_difexp, enable_difexp=enable_difexp,
lfc_cutoff=lfc_cutoff, lfc_cutoff=lfc_cutoff,
top_n=top_n, top_n=top_n,
X_approx_distribution=X_approx_distribution, X_approximate_distribution=X_approximate_distribution,
config_file_name=f"temp_dataset_config_{random_num}.yml", config_file_name=f"temp_dataset_config_{random_num}.yml",
) )
external_config = self.custom_external_config( external_config = self.custom_external_config(
@@ -231,7 +231,7 @@ class ConfigTests(BaseTest):
enable_difexp="true", enable_difexp="true",
lfc_cutoff=0.01, lfc_cutoff=0.01,
top_n=10, top_n=10,
X_approx_distribution="normal", X_approximate_distribution="normal",
config_file_name="dataset_config.yml", config_file_name="dataset_config.yml",
): ):
configfile = os.path.join(self.tmp_fixtures_directory, config_file_name) configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
@@ -20,7 +20,7 @@ class DiffExpTest(unittest.TestCase):
adaptor types and different algorithms.""" adaptor types and different algorithms."""
def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}): def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}):
extra_dataset_config["X_approx_distribution"] = "normal" # hardwired for now extra_dataset_config["X_approximate_distribution"] = "normal" # hardwired for now
config = app_config(path, extra_server_config=extra_server_config, extra_dataset_config=extra_dataset_config) config = app_config(path, extra_server_config=extra_server_config, extra_dataset_config=extra_dataset_config)
loader = MatrixDataLoader(path) loader = MatrixDataLoader(path)
adaptor = loader.open(config) adaptor = loader.open(config)
@@ -103,7 +103,7 @@ class ConfigTests(unittest.TestCase):
environment=None, environment=None,
aws_secrets_manager_region=None, aws_secrets_manager_region=None,
aws_secrets_manager_secrets=[], aws_secrets_manager_secrets=[],
X_approx_distribution="auto", X_approximate_distribution="auto",
config_file_name="app_config.yml", config_file_name="app_config.yml",
): ):
random_num = random.randrange(999999) random_num = random.randrange(999999)
@@ -151,7 +151,7 @@ class ConfigTests(unittest.TestCase):
enable_difexp=enable_difexp, enable_difexp=enable_difexp,
lfc_cutoff=lfc_cutoff, lfc_cutoff=lfc_cutoff,
top_n=top_n, top_n=top_n,
X_approx_distribution=X_approx_distribution, X_approximate_distribution=X_approximate_distribution,
config_file_name=f"temp_dataset_config_{random_num}.yml", config_file_name=f"temp_dataset_config_{random_num}.yml",
) )
external_config = self.custom_external_config( external_config = self.custom_external_config(
@@ -187,7 +187,7 @@ class ConfigTests(unittest.TestCase):
enable_difexp="true", enable_difexp="true",
lfc_cutoff=0.01, lfc_cutoff=0.01,
top_n=10, top_n=10,
X_approx_distribution="auto", X_approximate_distribution="auto",
config_file_name="dataset_config.yml", config_file_name="dataset_config.yml",
): ):
configfile = os.path.join(self.tmp_fixtures_directory, config_file_name) configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
@@ -2,7 +2,7 @@ import unittest
import numpy as np import numpy as np
from scipy import sparse from scipy import sparse
from backend.common.compute.estimate_distribution import estimate_approximate_distribution from backend.common.compute.estimate_distribution import estimate_approximate_distribution
from backend.common.constants import XApproxDistribution from backend.common.constants import XApproximateDistribution
from backend.server.data_common.matrix_loader import MatrixDataLoader from backend.server.data_common.matrix_loader import MatrixDataLoader
from backend.test.test_server.unit import app_config from backend.test.test_server.unit import app_config
from backend.test import PROJECT_ROOT from backend.test import PROJECT_ROOT
@@ -20,22 +20,22 @@ class EstDistTest(unittest.TestCase):
def test_adaptestimate_approximate_distribution(self): def test_adaptestimate_approximate_distribution(self):
adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad") adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
self.assertEqual(adaptor.get_X_approx_distribution(), XApproxDistribution.NORMAL) self.assertEqual(adaptor.get_X_approximate_distribution(), XApproximateDistribution.NORMAL)
def test_estimate_approximate_distribution(self): def test_estimate_approximate_distribution(self):
raw = np.random.exponential(scale=1000, size=(100, 40)) raw = np.random.exponential(scale=1000, size=(100, 40))
# ndarray # ndarray
self.assertEqual(estimate_approximate_distribution(raw), XApproxDistribution.COUNT) self.assertEqual(estimate_approximate_distribution(raw), XApproximateDistribution.COUNT)
self.assertEqual(estimate_approximate_distribution(np.log1p(raw)), XApproxDistribution.NORMAL) self.assertEqual(estimate_approximate_distribution(np.log1p(raw)), XApproximateDistribution.NORMAL)
# csr_matrix # csr_matrix
self.assertEqual(estimate_approximate_distribution(sparse.csr_matrix(raw)), XApproxDistribution.COUNT) self.assertEqual(estimate_approximate_distribution(sparse.csr_matrix(raw)), XApproximateDistribution.COUNT)
self.assertEqual( self.assertEqual(
estimate_approximate_distribution(sparse.csr_matrix(np.log1p(raw))), XApproxDistribution.NORMAL estimate_approximate_distribution(sparse.csr_matrix(np.log1p(raw))), XApproximateDistribution.NORMAL
) )
# BIG (ie, trigger MT) # BIG (ie, trigger MT)
big = np.random.exponential(scale=100, size=(1_000_000, 100)) big = np.random.exponential(scale=100, size=(1_000_000, 100))
self.assertEqual(estimate_approximate_distribution(big), XApproxDistribution.COUNT) self.assertEqual(estimate_approximate_distribution(big), XApproximateDistribution.COUNT)
self.assertEqual(estimate_approximate_distribution(np.log1p(big)), XApproxDistribution.NORMAL) self.assertEqual(estimate_approximate_distribution(np.log1p(big)), XApproximateDistribution.NORMAL)
@@ -22,7 +22,7 @@ Test the anndata adaptor using the pbmc3k data set.
@parameterized_class( @parameterized_class(
("data_locator", "backed", "X_approx_distribution"), ("data_locator", "backed", "X_approximate_distribution"),
[ [
(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False, "auto"), (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False, "auto"),
(f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", False, "auto"), (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", False, "auto"),
@@ -41,7 +41,9 @@ Test the anndata adaptor using the pbmc3k data set.
class AdaptorTest(unittest.TestCase): class AdaptorTest(unittest.TestCase):
def setUp(self): def setUp(self):
config = app_config( config = app_config(
self.data_locator, self.backed, extra_dataset_config=dict(X_approx_distribution=self.X_approx_distribution) self.data_locator,
self.backed,
extra_dataset_config=dict(X_approximate_distribution=self.X_approximate_distribution),
) )
self.data = AnndataAdaptor(DataLocator(self.data_locator), config) self.data = AnndataAdaptor(DataLocator(self.data_locator), config)