mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-10-07 21:28:11 +08:00
rename X_approx_distribution to X_approximate_distribution (#2337)
This commit is contained in:
@@ -1,6 +1,6 @@
|
|||||||
import numpy as np
|
import numpy as np
|
||||||
from scipy import sparse, stats
|
from scipy import sparse, stats
|
||||||
from backend.common.constants import XApproxDistribution
|
from backend.common.constants import XApproximateDistribution
|
||||||
|
|
||||||
|
|
||||||
def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
|
def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
|
||||||
@@ -30,13 +30,13 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
|
|||||||
:return: for top N genes, {"positive": for top N genes, [ varindex, foldchange, pval, pval_adj ], "negative": for top N genes, [ varindex, foldchange, pval, pval_adj ]}
|
:return: for top N genes, {"positive": for top N genes, [ varindex, foldchange, pval, pval_adj ], "negative": for top N genes, [ varindex, foldchange, pval, pval_adj ]}
|
||||||
"""
|
"""
|
||||||
|
|
||||||
X_approx_distribution = adaptor.get_X_approx_distribution()
|
X_approximate_distribution = adaptor.get_X_approximate_distribution()
|
||||||
dataA = adaptor.get_X_array(maskA, None)
|
dataA = adaptor.get_X_array(maskA, None)
|
||||||
dataB = adaptor.get_X_array(maskB, None)
|
dataB = adaptor.get_X_array(maskB, None)
|
||||||
|
|
||||||
# mean, variance, N - calculate for both selections
|
# mean, variance, N - calculate for both selections
|
||||||
meanA, vA, nA = mean_var_n(dataA, X_approx_distribution)
|
meanA, vA, nA = mean_var_n(dataA, X_approximate_distribution)
|
||||||
meanB, vB, nB = mean_var_n(dataB, X_approx_distribution)
|
meanB, vB, nB = mean_var_n(dataB, X_approximate_distribution)
|
||||||
res = diffexp_ttest_from_mean_var(meanA, vA, nA, meanB, vB, nB, top_n, diffexp_lfc_cutoff)
|
res = diffexp_ttest_from_mean_var(meanA, vA, nA, meanB, vB, nB, top_n, diffexp_lfc_cutoff)
|
||||||
|
|
||||||
return res
|
return res
|
||||||
@@ -113,7 +113,7 @@ def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp
|
|||||||
|
|
||||||
|
|
||||||
# Convenience function which handles sparse data
|
# Convenience function which handles sparse data
|
||||||
def mean_var_n(X, X_approx_distribution=XApproxDistribution.NORMAL):
|
def mean_var_n(X, X_approximate_distribution=XApproximateDistribution.NORMAL):
|
||||||
"""
|
"""
|
||||||
Two-pass variance calculation. Numerically (more) stable
|
Two-pass variance calculation. Numerically (more) stable
|
||||||
than naive methods (and same method used by numpy.var())
|
than naive methods (and same method used by numpy.var())
|
||||||
@@ -131,14 +131,14 @@ def mean_var_n(X, X_approx_distribution=XApproxDistribution.NORMAL):
|
|||||||
with np.errstate(divide="call", invalid="call", call=fp_err_set):
|
with np.errstate(divide="call", invalid="call", call=fp_err_set):
|
||||||
n = X.shape[0]
|
n = X.shape[0]
|
||||||
if sparse.issparse(X):
|
if sparse.issparse(X):
|
||||||
if X_approx_distribution == XApproxDistribution.COUNT:
|
if X_approximate_distribution == XApproximateDistribution.COUNT:
|
||||||
X = X.log1p()
|
X = X.log1p()
|
||||||
mean = X.mean(axis=0).A1
|
mean = X.mean(axis=0).A1
|
||||||
dfm = X - mean
|
dfm = X - mean
|
||||||
sumsq = np.sum(np.multiply(dfm, dfm), axis=0).A1
|
sumsq = np.sum(np.multiply(dfm, dfm), axis=0).A1
|
||||||
v = sumsq / (n - 1)
|
v = sumsq / (n - 1)
|
||||||
else:
|
else:
|
||||||
if X_approx_distribution == XApproxDistribution.COUNT:
|
if X_approximate_distribution == XApproximateDistribution.COUNT:
|
||||||
X = np.log1p(X)
|
X = np.log1p(X)
|
||||||
mean = X.mean(axis=0)
|
mean = X.mean(axis=0)
|
||||||
dfm = X - mean
|
dfm = X - mean
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import numba
|
|||||||
import concurrent.futures
|
import concurrent.futures
|
||||||
import numpy as np
|
import numpy as np
|
||||||
from scipy import sparse
|
from scipy import sparse
|
||||||
from backend.common.constants import XApproxDistribution
|
from backend.common.constants import XApproximateDistribution
|
||||||
|
|
||||||
|
|
||||||
@numba.njit(fastmath=True, error_model="numpy", nogil=True)
|
@numba.njit(fastmath=True, error_model="numpy", nogil=True)
|
||||||
@@ -29,7 +29,7 @@ def min_max(arr):
|
|||||||
return min_val, max_val
|
return min_val, max_val
|
||||||
|
|
||||||
|
|
||||||
def estimate_approximate_distribution(X) -> XApproxDistribution:
|
def estimate_approximate_distribution(X) -> XApproximateDistribution:
|
||||||
"""
|
"""
|
||||||
Estimate the distribution (normal, count) of the X matrix.
|
Estimate the distribution (normal, count) of the X matrix.
|
||||||
|
|
||||||
@@ -59,4 +59,4 @@ def estimate_approximate_distribution(X) -> XApproxDistribution:
|
|||||||
min_val, max_val = min_max(Xdata)
|
min_val, max_val = min_max(Xdata)
|
||||||
|
|
||||||
excess_range = (max_val - min_val) > 24
|
excess_range = (max_val - min_val) > 24
|
||||||
return XApproxDistribution.COUNT if excess_range else XApproxDistribution.NORMAL
|
return XApproximateDistribution.COUNT if excess_range else XApproximateDistribution.NORMAL
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ class DiffExpMode(AugmentedEnum):
|
|||||||
VAR_FILTER = "varFilter"
|
VAR_FILTER = "varFilter"
|
||||||
|
|
||||||
|
|
||||||
class XApproxDistribution(AugmentedEnum):
|
class XApproximateDistribution(AugmentedEnum):
|
||||||
NORMAL = "normal"
|
NORMAL = "normal"
|
||||||
COUNT = "count"
|
COUNT = "count"
|
||||||
|
|
||||||
|
|||||||
@@ -44,7 +44,7 @@ class DatasetConfig(BaseConfig):
|
|||||||
self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
|
self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
|
||||||
self.diffexp__top_n = default_config["diffexp"]["top_n"]
|
self.diffexp__top_n = default_config["diffexp"]["top_n"]
|
||||||
|
|
||||||
self.X_approx_distribution = default_config["X_approx_distribution"]
|
self.X_approximate_distribution = default_config["X_approximate_distribution"]
|
||||||
|
|
||||||
except KeyError as e:
|
except KeyError as e:
|
||||||
raise ConfigurationError(f"Unexpected config: {str(e)}")
|
raise ConfigurationError(f"Unexpected config: {str(e)}")
|
||||||
@@ -60,7 +60,7 @@ class DatasetConfig(BaseConfig):
|
|||||||
self.handle_user_annotations(context)
|
self.handle_user_annotations(context)
|
||||||
self.handle_embeddings()
|
self.handle_embeddings()
|
||||||
self.handle_diffexp(context)
|
self.handle_diffexp(context)
|
||||||
self.handle_X_approx_distribution()
|
self.handle_X_approximate_distribution()
|
||||||
|
|
||||||
def handle_app(self):
|
def handle_app(self):
|
||||||
self.validate_correct_type_of_configuration_attribute("app__scripts", list)
|
self.validate_correct_type_of_configuration_attribute("app__scripts", list)
|
||||||
@@ -203,9 +203,9 @@ class DatasetConfig(BaseConfig):
|
|||||||
"running differential expression may take longer or fail."
|
"running differential expression may take longer or fail."
|
||||||
)
|
)
|
||||||
|
|
||||||
def handle_X_approx_distribution(self):
|
def handle_X_approximate_distribution(self):
|
||||||
self.validate_correct_type_of_configuration_attribute("X_approx_distribution", str)
|
self.validate_correct_type_of_configuration_attribute("X_approximate_distribution", str)
|
||||||
if self.X_approx_distribution not in ["normal", "count"]:
|
if self.X_approximate_distribution not in ["normal", "count"]:
|
||||||
raise ConfigurationError(
|
raise ConfigurationError(
|
||||||
"X_approx_distribution has unknown value -- must be 'normal' or 'count'."
|
"X_approximate_distribution has unknown value -- must be 'normal' or 'count'."
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from scipy import sparse
|
|||||||
|
|
||||||
import backend.common.compute.diffexp_generic as diffexp_generic
|
import backend.common.compute.diffexp_generic as diffexp_generic
|
||||||
from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
|
from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
|
||||||
from backend.common.constants import Axis, MAX_LAYOUTS, XApproxDistribution
|
from backend.common.constants import Axis, MAX_LAYOUTS, XApproximateDistribution
|
||||||
from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata
|
from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata
|
||||||
from backend.common.errors import PrepareError, DatasetAccessError, ConfigurationError
|
from backend.common.errors import PrepareError, DatasetAccessError, ConfigurationError
|
||||||
from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
|
from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
|
||||||
@@ -28,7 +28,7 @@ class AnndataAdaptor(DataAdaptor):
|
|||||||
def __init__(self, data_locator, app_config=None, dataset_config=None):
|
def __init__(self, data_locator, app_config=None, dataset_config=None):
|
||||||
super().__init__(data_locator, app_config, dataset_config)
|
super().__init__(data_locator, app_config, dataset_config)
|
||||||
self.data = None
|
self.data = None
|
||||||
self.X_approx_distribution = None
|
self.X_approximate_distribution = None
|
||||||
self._load_data(data_locator)
|
self._load_data(data_locator)
|
||||||
self._validate_and_initialize()
|
self._validate_and_initialize()
|
||||||
|
|
||||||
@@ -191,9 +191,9 @@ class AnndataAdaptor(DataAdaptor):
|
|||||||
self.gene_count = self.data.shape[1]
|
self.gene_count = self.data.shape[1]
|
||||||
self._create_schema()
|
self._create_schema()
|
||||||
|
|
||||||
if self.dataset_config.X_approx_distribution == "auto":
|
if self.dataset_config.X_approximate_distribution == "auto":
|
||||||
raise ConfigurationError("X-approx-distribution 'auto' mode unsupported.")
|
raise ConfigurationError("X-approximate-distribution 'auto' mode unsupported.")
|
||||||
self.X_approx_distribution = self.dataset_config.X_approx_distribution
|
self.X_approximate_distribution = self.dataset_config.X_approximate_distribution
|
||||||
|
|
||||||
# heuristic
|
# heuristic
|
||||||
n_values = self.data.shape[0] * self.data.shape[1]
|
n_values = self.data.shape[0] * self.data.shape[1]
|
||||||
@@ -327,8 +327,8 @@ class AnndataAdaptor(DataAdaptor):
|
|||||||
X = self.data.X[obs_mask, var_mask]
|
X = self.data.X[obs_mask, var_mask]
|
||||||
return X
|
return X
|
||||||
|
|
||||||
def get_X_approx_distribution(self) -> XApproxDistribution:
|
def get_X_approximate_distribution(self) -> XApproximateDistribution:
|
||||||
return self.X_approx_distribution
|
return self.X_approximate_distribution
|
||||||
|
|
||||||
def get_shape(self):
|
def get_shape(self):
|
||||||
return self.data.shape
|
return self.data.shape
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ from scipy import sparse
|
|||||||
from server_timing import Timing as ServerTiming
|
from server_timing import Timing as ServerTiming
|
||||||
|
|
||||||
from backend.czi_hosted.common.config.app_config import AppConfig
|
from backend.czi_hosted.common.config.app_config import AppConfig
|
||||||
from backend.common.constants import Axis, XApproxDistribution
|
from backend.common.constants import Axis, XApproximateDistribution
|
||||||
from backend.common.errors import (
|
from backend.common.errors import (
|
||||||
FilterError,
|
FilterError,
|
||||||
JSONEncodingValueError,
|
JSONEncodingValueError,
|
||||||
@@ -84,7 +84,7 @@ class DataAdaptor(metaclass=ABCMeta):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def get_X_approx_distribution(self) -> XApproxDistribution:
|
def get_X_approximate_distribution(self) -> XApproximateDistribution:
|
||||||
"""return the approximate distribution of the X matrix."""
|
"""return the approximate distribution of the X matrix."""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import pandas as pd
|
|||||||
import tiledb
|
import tiledb
|
||||||
from server_timing import Timing as ServerTiming
|
from server_timing import Timing as ServerTiming
|
||||||
|
|
||||||
from backend.common.constants import Axis, XApproxDistribution
|
from backend.common.constants import Axis, XApproximateDistribution
|
||||||
from backend.common.errors import DatasetAccessError, ConfigurationError
|
from backend.common.errors import DatasetAccessError, ConfigurationError
|
||||||
from backend.czi_hosted.common.immutable_kvcache import ImmutableKVCache
|
from backend.czi_hosted.common.immutable_kvcache import ImmutableKVCache
|
||||||
from backend.common.utils.type_conversion_utils import get_schema_type_hint_from_dtype
|
from backend.common.utils.type_conversion_utils import get_schema_type_hint_from_dtype
|
||||||
@@ -37,7 +37,7 @@ class CxgAdaptor(DataAdaptor):
|
|||||||
self.lsuri_results = ImmutableKVCache(lambda key: self._lsuri(uri=key, tiledb_ctx=self.tiledb_ctx))
|
self.lsuri_results = ImmutableKVCache(lambda key: self._lsuri(uri=key, tiledb_ctx=self.tiledb_ctx))
|
||||||
self.arrays = ImmutableKVCache(lambda key: self._open_array(uri=key, tiledb_ctx=self.tiledb_ctx))
|
self.arrays = ImmutableKVCache(lambda key: self._open_array(uri=key, tiledb_ctx=self.tiledb_ctx))
|
||||||
self.schema = None
|
self.schema = None
|
||||||
self.X_approx_distribution = None
|
self.X_approximate_distribution = None
|
||||||
|
|
||||||
self._validate_and_initialize()
|
self._validate_and_initialize()
|
||||||
|
|
||||||
@@ -176,9 +176,9 @@ class CxgAdaptor(DataAdaptor):
|
|||||||
if cxg_version not in ["0.0", "0.1", "0.2.0"]:
|
if cxg_version not in ["0.0", "0.1", "0.2.0"]:
|
||||||
raise DatasetAccessError(f"cxg matrix is not valid: {self.url}")
|
raise DatasetAccessError(f"cxg matrix is not valid: {self.url}")
|
||||||
|
|
||||||
if self.dataset_config.X_approx_distribution == "auto":
|
if self.dataset_config.X_approximate_distribution == "auto":
|
||||||
raise ConfigurationError("X-approx-distribution 'auto' mode unsupported.")
|
raise ConfigurationError("X-approximate-distribution 'auto' mode unsupported.")
|
||||||
self.X_approx_distribution = self.dataset_config.X_approx_distribution
|
self.X_approximate_distribution = self.dataset_config.X_approximate_distribution
|
||||||
|
|
||||||
self.title = title
|
self.title = title
|
||||||
self.about = about
|
self.about = about
|
||||||
@@ -286,8 +286,8 @@ class CxgAdaptor(DataAdaptor):
|
|||||||
data = X.multi_index[obs_items, var_items][""]
|
data = X.multi_index[obs_items, var_items][""]
|
||||||
return data
|
return data
|
||||||
|
|
||||||
def get_X_approx_distribution(self) -> XApproxDistribution:
|
def get_X_approximate_distribution(self) -> XApproximateDistribution:
|
||||||
return self.X_approx_distribution
|
return self.X_approximate_distribution
|
||||||
|
|
||||||
def get_shape(self):
|
def get_shape(self):
|
||||||
X = self.open_array("X")
|
X = self.open_array("X")
|
||||||
|
|||||||
@@ -203,7 +203,7 @@ dataset:
|
|||||||
lfc_cutoff: 0.01
|
lfc_cutoff: 0.01
|
||||||
top_n: 10
|
top_n: 10
|
||||||
|
|
||||||
X_approx_distribution: normal # currently fixed config
|
X_approximate_distribution: normal # currently fixed config
|
||||||
|
|
||||||
external:
|
external:
|
||||||
# You can retrieve configuration parameters from this config file, the environment,
|
# You can retrieve configuration parameters from this config file, the environment,
|
||||||
|
|||||||
@@ -151,8 +151,8 @@ def dataset_args(func):
|
|||||||
help="URL providing more information about the dataset (hint: must be a fully specified absolute URL).",
|
help="URL providing more information about the dataset (hint: must be a fully specified absolute URL).",
|
||||||
)
|
)
|
||||||
@click.option(
|
@click.option(
|
||||||
"--X-approx-distribution",
|
"--X-approximate-distribution",
|
||||||
default=DEFAULT_CONFIG.dataset_config.X_approx_distribution,
|
default=DEFAULT_CONFIG.dataset_config.X_approximate_distribution,
|
||||||
show_default=True,
|
show_default=True,
|
||||||
type=click.Choice(["auto", "normal", "count"], case_sensitive=False),
|
type=click.Choice(["auto", "normal", "count"], case_sensitive=False),
|
||||||
help="Specify the approximate distribution of X matrix values. 'auto' will use a heuristic "
|
help="Specify the approximate distribution of X matrix values. 'auto' will use a heuristic "
|
||||||
@@ -326,7 +326,7 @@ def launch(
|
|||||||
disable_diffexp,
|
disable_diffexp,
|
||||||
config_file,
|
config_file,
|
||||||
dump_default_config,
|
dump_default_config,
|
||||||
x_approx_distribution,
|
x_approximate_distribution,
|
||||||
):
|
):
|
||||||
"""Launch the cellxgene data viewer.
|
"""Launch the cellxgene data viewer.
|
||||||
This web app lets you explore single-cell expression data.
|
This web app lets you explore single-cell expression data.
|
||||||
@@ -385,7 +385,7 @@ def launch(
|
|||||||
embeddings__names=embedding,
|
embeddings__names=embedding,
|
||||||
diffexp__enable=not disable_diffexp,
|
diffexp__enable=not disable_diffexp,
|
||||||
diffexp__lfc_cutoff=diffexp_lfc_cutoff,
|
diffexp__lfc_cutoff=diffexp_lfc_cutoff,
|
||||||
X_approx_distribution=x_approx_distribution,
|
X_approximate_distribution=x_approximate_distribution,
|
||||||
)
|
)
|
||||||
|
|
||||||
diff = cli_config.server_config.changes_from_default()
|
diff = cli_config.server_config.changes_from_default()
|
||||||
|
|||||||
@@ -38,7 +38,7 @@ class DatasetConfig(BaseConfig):
|
|||||||
self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
|
self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
|
||||||
self.diffexp__top_n = default_config["diffexp"]["top_n"]
|
self.diffexp__top_n = default_config["diffexp"]["top_n"]
|
||||||
|
|
||||||
self.X_approx_distribution = default_config["X_approx_distribution"]
|
self.X_approximate_distribution = default_config["X_approximate_distribution"]
|
||||||
|
|
||||||
except KeyError as e:
|
except KeyError as e:
|
||||||
raise ConfigurationError(f"Unexpected config: {str(e)}")
|
raise ConfigurationError(f"Unexpected config: {str(e)}")
|
||||||
@@ -52,7 +52,7 @@ class DatasetConfig(BaseConfig):
|
|||||||
self.handle_user_annotations(context)
|
self.handle_user_annotations(context)
|
||||||
self.handle_embeddings()
|
self.handle_embeddings()
|
||||||
self.handle_diffexp(context)
|
self.handle_diffexp(context)
|
||||||
self.handle_X_approx_distribution()
|
self.handle_X_approximate_distribution()
|
||||||
|
|
||||||
def get_data_adaptor(self):
|
def get_data_adaptor(self):
|
||||||
server_config = self.app_config.server_config
|
server_config = self.app_config.server_config
|
||||||
@@ -186,9 +186,9 @@ class DatasetConfig(BaseConfig):
|
|||||||
"CAUTION: due to the size of your dataset, " "running differential expression may take longer or fail."
|
"CAUTION: due to the size of your dataset, " "running differential expression may take longer or fail."
|
||||||
)
|
)
|
||||||
|
|
||||||
def handle_X_approx_distribution(self):
|
def handle_X_approximate_distribution(self):
|
||||||
self.validate_correct_type_of_configuration_attribute("X_approx_distribution", str)
|
self.validate_correct_type_of_configuration_attribute("X_approximate_distribution", str)
|
||||||
if self.X_approx_distribution not in ["auto", "normal", "count"]:
|
if self.X_approximate_distribution not in ["auto", "normal", "count"]:
|
||||||
raise ConfigurationError(
|
raise ConfigurationError(
|
||||||
"X_approx_distribution has unknown value -- must be 'auto', 'normal' or 'count'."
|
"X_approximate_distribution has unknown value -- must be 'auto', 'normal' or 'count'."
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ from scipy import sparse
|
|||||||
import backend.common.compute.diffexp_generic as diffexp_generic
|
import backend.common.compute.diffexp_generic as diffexp_generic
|
||||||
import backend.common.compute.estimate_distribution as estimate_distribution
|
import backend.common.compute.estimate_distribution as estimate_distribution
|
||||||
from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
|
from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
|
||||||
from backend.common.constants import Axis, MAX_LAYOUTS, XApproxDistribution
|
from backend.common.constants import Axis, MAX_LAYOUTS, XApproximateDistribution
|
||||||
from backend.server.common.corpora import corpora_get_props_from_anndata
|
from backend.server.common.corpora import corpora_get_props_from_anndata
|
||||||
from backend.common.errors import PrepareError, DatasetAccessError
|
from backend.common.errors import PrepareError, DatasetAccessError
|
||||||
from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
|
from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
|
||||||
@@ -29,7 +29,7 @@ class AnndataAdaptor(DataAdaptor):
|
|||||||
def __init__(self, data_locator, app_config=None, dataset_config=None):
|
def __init__(self, data_locator, app_config=None, dataset_config=None):
|
||||||
super().__init__(data_locator, app_config, dataset_config)
|
super().__init__(data_locator, app_config, dataset_config)
|
||||||
self.data = None
|
self.data = None
|
||||||
self.X_approx_distribution = None
|
self.X_approximate_distribution = None
|
||||||
self._load_data(data_locator)
|
self._load_data(data_locator)
|
||||||
self._validate_and_initialize()
|
self._validate_and_initialize()
|
||||||
|
|
||||||
@@ -192,12 +192,12 @@ class AnndataAdaptor(DataAdaptor):
|
|||||||
self.gene_count = self.data.shape[1]
|
self.gene_count = self.data.shape[1]
|
||||||
self._create_schema()
|
self._create_schema()
|
||||||
|
|
||||||
if self.dataset_config.X_approx_distribution == "auto":
|
if self.dataset_config.X_approximate_distribution == "auto":
|
||||||
"""Lazy evaluate the heuristic if we are backed."""
|
"""Lazy evaluate the heuristic if we are backed."""
|
||||||
if not self.data.isbacked:
|
if not self.data.isbacked:
|
||||||
self.X_approx_distribution = estimate_distribution.estimate_approximate_distribution(self.data.X)
|
self.X_approximate_distribution = estimate_distribution.estimate_approximate_distribution(self.data.X)
|
||||||
else:
|
else:
|
||||||
self.X_approx_distribution = self.dataset_config.X_approx_distribution
|
self.X_approximate_distribution = self.dataset_config.X_approximate_distribution
|
||||||
|
|
||||||
# heuristic
|
# heuristic
|
||||||
n_values = self.data.shape[0] * self.data.shape[1]
|
n_values = self.data.shape[0] * self.data.shape[1]
|
||||||
@@ -331,15 +331,15 @@ class AnndataAdaptor(DataAdaptor):
|
|||||||
X = self.data.X[obs_mask, var_mask]
|
X = self.data.X[obs_mask, var_mask]
|
||||||
return X
|
return X
|
||||||
|
|
||||||
def get_X_approx_distribution(self) -> XApproxDistribution:
|
def get_X_approximate_distribution(self) -> XApproximateDistribution:
|
||||||
"""return the approximate distribution of the X matrix."""
|
"""return the approximate distribution of the X matrix."""
|
||||||
if self.X_approx_distribution is None:
|
if self.X_approximate_distribution is None:
|
||||||
"""Not yet evaluated."""
|
"""Not yet evaluated."""
|
||||||
assert(self.dataset_config.X_approx_distribution == "auto")
|
assert(self.dataset_config.X_approximate_distribution == "auto")
|
||||||
self.data = self.data.to_memory() # loads data
|
self.data = self.data.to_memory() # loads data
|
||||||
self.X_approx_distribution = estimate_distribution.estimate_approximate_distribution(self.data.X)
|
self.X_approximate_distribution = estimate_distribution.estimate_approximate_distribution(self.data.X)
|
||||||
|
|
||||||
return self.X_approx_distribution
|
return self.X_approximate_distribution
|
||||||
|
|
||||||
def get_shape(self):
|
def get_shape(self):
|
||||||
return self.data.shape
|
return self.data.shape
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ from scipy import sparse
|
|||||||
from server_timing import Timing as ServerTiming
|
from server_timing import Timing as ServerTiming
|
||||||
|
|
||||||
from backend.server.common.config.app_config import AppConfig
|
from backend.server.common.config.app_config import AppConfig
|
||||||
from backend.common.constants import Axis, XApproxDistribution
|
from backend.common.constants import Axis, XApproximateDistribution
|
||||||
from backend.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError, UnsupportedSummaryMethod
|
from backend.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError, UnsupportedSummaryMethod
|
||||||
from backend.common.utils.utils import jsonify_numpy
|
from backend.common.utils.utils import jsonify_numpy
|
||||||
from backend.common.fbs.matrix import encode_matrix_fbs
|
from backend.common.fbs.matrix import encode_matrix_fbs
|
||||||
@@ -72,9 +72,9 @@ class DataAdaptor(metaclass=ABCMeta):
|
|||||||
the return type is either ndarray or scipy.sparse.spmatrix."""
|
the return type is either ndarray or scipy.sparse.spmatrix."""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def get_X_approx_distribution(self) -> XApproxDistribution:
|
def get_X_approximate_distribution(self) -> XApproximateDistribution:
|
||||||
"""return the approximate distribution of the X matrix."""
|
"""return the approximate distribution of the X matrix."""
|
||||||
return XApproxDistribution.NORMAL
|
return XApproximateDistribution.NORMAL
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def get_shape(self):
|
def get_shape(self):
|
||||||
|
|||||||
@@ -77,7 +77,7 @@ dataset:
|
|||||||
lfc_cutoff: 0.01
|
lfc_cutoff: 0.01
|
||||||
top_n: 10
|
top_n: 10
|
||||||
|
|
||||||
X_approx_distribution: auto
|
X_approximate_distribution: auto
|
||||||
|
|
||||||
external:
|
external:
|
||||||
# You can retrieve configuration parameters from this config file, the environment,
|
# You can retrieve configuration parameters from this config file, the environment,
|
||||||
|
|||||||
@@ -31,5 +31,5 @@ dataset:
|
|||||||
lfc_cutoff: {lfc_cutoff}
|
lfc_cutoff: {lfc_cutoff}
|
||||||
top_n: {top_n}
|
top_n: {top_n}
|
||||||
|
|
||||||
X_approx_distribution: {X_approx_distribution}
|
X_approximate_distribution: {X_approximate_distribution}
|
||||||
"""
|
"""
|
||||||
|
|||||||
+1
-1
@@ -28,5 +28,5 @@ dataset:
|
|||||||
lfc_cutoff: {lfc_cutoff}
|
lfc_cutoff: {lfc_cutoff}
|
||||||
top_n: {top_n}
|
top_n: {top_n}
|
||||||
|
|
||||||
X_approx_distribution: {X_approx_distribution}
|
X_approximate_distribution: {X_approximate_distribution}
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -131,7 +131,7 @@ class ConfigTests(BaseTest):
|
|||||||
environment=None,
|
environment=None,
|
||||||
aws_secrets_manager_region=None,
|
aws_secrets_manager_region=None,
|
||||||
aws_secrets_manager_secrets=[],
|
aws_secrets_manager_secrets=[],
|
||||||
X_approx_distribution="normal",
|
X_approximate_distribution="normal",
|
||||||
config_file_name="app_config.yml",
|
config_file_name="app_config.yml",
|
||||||
):
|
):
|
||||||
random_num = random.randrange(999999)
|
random_num = random.randrange(999999)
|
||||||
@@ -195,7 +195,7 @@ class ConfigTests(BaseTest):
|
|||||||
enable_difexp=enable_difexp,
|
enable_difexp=enable_difexp,
|
||||||
lfc_cutoff=lfc_cutoff,
|
lfc_cutoff=lfc_cutoff,
|
||||||
top_n=top_n,
|
top_n=top_n,
|
||||||
X_approx_distribution=X_approx_distribution,
|
X_approximate_distribution=X_approximate_distribution,
|
||||||
config_file_name=f"temp_dataset_config_{random_num}.yml",
|
config_file_name=f"temp_dataset_config_{random_num}.yml",
|
||||||
)
|
)
|
||||||
external_config = self.custom_external_config(
|
external_config = self.custom_external_config(
|
||||||
@@ -231,7 +231,7 @@ class ConfigTests(BaseTest):
|
|||||||
enable_difexp="true",
|
enable_difexp="true",
|
||||||
lfc_cutoff=0.01,
|
lfc_cutoff=0.01,
|
||||||
top_n=10,
|
top_n=10,
|
||||||
X_approx_distribution="normal",
|
X_approximate_distribution="normal",
|
||||||
config_file_name="dataset_config.yml",
|
config_file_name="dataset_config.yml",
|
||||||
):
|
):
|
||||||
configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
|
configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ class DiffExpTest(unittest.TestCase):
|
|||||||
adaptor types and different algorithms."""
|
adaptor types and different algorithms."""
|
||||||
|
|
||||||
def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}):
|
def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}):
|
||||||
extra_dataset_config["X_approx_distribution"] = "normal" # hardwired for now
|
extra_dataset_config["X_approximate_distribution"] = "normal" # hardwired for now
|
||||||
config = app_config(path, extra_server_config=extra_server_config, extra_dataset_config=extra_dataset_config)
|
config = app_config(path, extra_server_config=extra_server_config, extra_dataset_config=extra_dataset_config)
|
||||||
loader = MatrixDataLoader(path)
|
loader = MatrixDataLoader(path)
|
||||||
adaptor = loader.open(config)
|
adaptor = loader.open(config)
|
||||||
|
|||||||
@@ -103,7 +103,7 @@ class ConfigTests(unittest.TestCase):
|
|||||||
environment=None,
|
environment=None,
|
||||||
aws_secrets_manager_region=None,
|
aws_secrets_manager_region=None,
|
||||||
aws_secrets_manager_secrets=[],
|
aws_secrets_manager_secrets=[],
|
||||||
X_approx_distribution="auto",
|
X_approximate_distribution="auto",
|
||||||
config_file_name="app_config.yml",
|
config_file_name="app_config.yml",
|
||||||
):
|
):
|
||||||
random_num = random.randrange(999999)
|
random_num = random.randrange(999999)
|
||||||
@@ -151,7 +151,7 @@ class ConfigTests(unittest.TestCase):
|
|||||||
enable_difexp=enable_difexp,
|
enable_difexp=enable_difexp,
|
||||||
lfc_cutoff=lfc_cutoff,
|
lfc_cutoff=lfc_cutoff,
|
||||||
top_n=top_n,
|
top_n=top_n,
|
||||||
X_approx_distribution=X_approx_distribution,
|
X_approximate_distribution=X_approximate_distribution,
|
||||||
config_file_name=f"temp_dataset_config_{random_num}.yml",
|
config_file_name=f"temp_dataset_config_{random_num}.yml",
|
||||||
)
|
)
|
||||||
external_config = self.custom_external_config(
|
external_config = self.custom_external_config(
|
||||||
@@ -187,7 +187,7 @@ class ConfigTests(unittest.TestCase):
|
|||||||
enable_difexp="true",
|
enable_difexp="true",
|
||||||
lfc_cutoff=0.01,
|
lfc_cutoff=0.01,
|
||||||
top_n=10,
|
top_n=10,
|
||||||
X_approx_distribution="auto",
|
X_approximate_distribution="auto",
|
||||||
config_file_name="dataset_config.yml",
|
config_file_name="dataset_config.yml",
|
||||||
):
|
):
|
||||||
configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
|
configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import unittest
|
|||||||
import numpy as np
|
import numpy as np
|
||||||
from scipy import sparse
|
from scipy import sparse
|
||||||
from backend.common.compute.estimate_distribution import estimate_approximate_distribution
|
from backend.common.compute.estimate_distribution import estimate_approximate_distribution
|
||||||
from backend.common.constants import XApproxDistribution
|
from backend.common.constants import XApproximateDistribution
|
||||||
from backend.server.data_common.matrix_loader import MatrixDataLoader
|
from backend.server.data_common.matrix_loader import MatrixDataLoader
|
||||||
from backend.test.test_server.unit import app_config
|
from backend.test.test_server.unit import app_config
|
||||||
from backend.test import PROJECT_ROOT
|
from backend.test import PROJECT_ROOT
|
||||||
@@ -20,22 +20,22 @@ class EstDistTest(unittest.TestCase):
|
|||||||
|
|
||||||
def test_adaptestimate_approximate_distribution(self):
|
def test_adaptestimate_approximate_distribution(self):
|
||||||
adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
|
adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
|
||||||
self.assertEqual(adaptor.get_X_approx_distribution(), XApproxDistribution.NORMAL)
|
self.assertEqual(adaptor.get_X_approximate_distribution(), XApproximateDistribution.NORMAL)
|
||||||
|
|
||||||
def test_estimate_approximate_distribution(self):
|
def test_estimate_approximate_distribution(self):
|
||||||
raw = np.random.exponential(scale=1000, size=(100, 40))
|
raw = np.random.exponential(scale=1000, size=(100, 40))
|
||||||
|
|
||||||
# ndarray
|
# ndarray
|
||||||
self.assertEqual(estimate_approximate_distribution(raw), XApproxDistribution.COUNT)
|
self.assertEqual(estimate_approximate_distribution(raw), XApproximateDistribution.COUNT)
|
||||||
self.assertEqual(estimate_approximate_distribution(np.log1p(raw)), XApproxDistribution.NORMAL)
|
self.assertEqual(estimate_approximate_distribution(np.log1p(raw)), XApproximateDistribution.NORMAL)
|
||||||
|
|
||||||
# csr_matrix
|
# csr_matrix
|
||||||
self.assertEqual(estimate_approximate_distribution(sparse.csr_matrix(raw)), XApproxDistribution.COUNT)
|
self.assertEqual(estimate_approximate_distribution(sparse.csr_matrix(raw)), XApproximateDistribution.COUNT)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
estimate_approximate_distribution(sparse.csr_matrix(np.log1p(raw))), XApproxDistribution.NORMAL
|
estimate_approximate_distribution(sparse.csr_matrix(np.log1p(raw))), XApproximateDistribution.NORMAL
|
||||||
)
|
)
|
||||||
|
|
||||||
# BIG (ie, trigger MT)
|
# BIG (ie, trigger MT)
|
||||||
big = np.random.exponential(scale=100, size=(1_000_000, 100))
|
big = np.random.exponential(scale=100, size=(1_000_000, 100))
|
||||||
self.assertEqual(estimate_approximate_distribution(big), XApproxDistribution.COUNT)
|
self.assertEqual(estimate_approximate_distribution(big), XApproximateDistribution.COUNT)
|
||||||
self.assertEqual(estimate_approximate_distribution(np.log1p(big)), XApproxDistribution.NORMAL)
|
self.assertEqual(estimate_approximate_distribution(np.log1p(big)), XApproximateDistribution.NORMAL)
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ Test the anndata adaptor using the pbmc3k data set.
|
|||||||
|
|
||||||
|
|
||||||
@parameterized_class(
|
@parameterized_class(
|
||||||
("data_locator", "backed", "X_approx_distribution"),
|
("data_locator", "backed", "X_approximate_distribution"),
|
||||||
[
|
[
|
||||||
(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False, "auto"),
|
(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False, "auto"),
|
||||||
(f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", False, "auto"),
|
(f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", False, "auto"),
|
||||||
@@ -41,7 +41,9 @@ Test the anndata adaptor using the pbmc3k data set.
|
|||||||
class AdaptorTest(unittest.TestCase):
|
class AdaptorTest(unittest.TestCase):
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
config = app_config(
|
config = app_config(
|
||||||
self.data_locator, self.backed, extra_dataset_config=dict(X_approx_distribution=self.X_approx_distribution)
|
self.data_locator,
|
||||||
|
self.backed,
|
||||||
|
extra_dataset_config=dict(X_approximate_distribution=self.X_approximate_distribution),
|
||||||
)
|
)
|
||||||
self.data = AnndataAdaptor(DataLocator(self.data_locator), config)
|
self.data = AnndataAdaptor(DataLocator(self.data_locator), config)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user