fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-15 12:47:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

rename X_approx_distribution to X_approximate_distribution (#2337)

Browse Source
This commit is contained in:
Bruce Martin
2021-07-27 13:43:04 -07:00
committed by GitHub
parent 1998c0ad63
commit 0b1ab02a60
20 changed files with 79 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
backend/common/compute/diffexp_generic.py
View File

@@ -1,6 +1,6 @@
import numpy as np
from scipy import sparse, stats
from backend.common.constants import XApproxDistribution
from backend.common.constants import XApproximateDistribution
def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
@@ -30,13 +30,13 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
:return: for top N genes, {"positive": for top N genes, [ varindex, foldchange, pval, pval_adj ], "negative": for top N genes, [ varindex, foldchange, pval, pval_adj ]}
"""
X_approx_distribution = adaptor.get_X_approx_distribution()
X_approximate_distribution = adaptor.get_X_approximate_distribution()
dataA = adaptor.get_X_array(maskA, None)
dataB = adaptor.get_X_array(maskB, None)
# mean, variance, N - calculate for both selections
meanA, vA, nA = mean_var_n(dataA, X_approx_distribution)
meanB, vB, nB = mean_var_n(dataB, X_approx_distribution)
meanA, vA, nA = mean_var_n(dataA, X_approximate_distribution)
meanB, vB, nB = mean_var_n(dataB, X_approximate_distribution)
res = diffexp_ttest_from_mean_var(meanA, vA, nA, meanB, vB, nB, top_n, diffexp_lfc_cutoff)
return res
@@ -113,7 +113,7 @@ def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp
# Convenience function which handles sparse data
def mean_var_n(X, X_approx_distribution=XApproxDistribution.NORMAL):
def mean_var_n(X, X_approximate_distribution=XApproximateDistribution.NORMAL):
"""
Two-pass variance calculation. Numerically (more) stable
than naive methods (and same method used by numpy.var())
@@ -131,14 +131,14 @@ def mean_var_n(X, X_approx_distribution=XApproxDistribution.NORMAL):
with np.errstate(divide="call", invalid="call", call=fp_err_set):
n = X.shape[0]
if sparse.issparse(X):
if X_approx_distribution == XApproxDistribution.COUNT:
if X_approximate_distribution == XApproximateDistribution.COUNT:
X = X.log1p()
mean = X.mean(axis=0).A1
dfm = X - mean
sumsq = np.sum(np.multiply(dfm, dfm), axis=0).A1
v = sumsq / (n - 1)
else:
if X_approx_distribution == XApproxDistribution.COUNT:
if X_approximate_distribution == XApproximateDistribution.COUNT:
X = np.log1p(X)
mean = X.mean(axis=0)
dfm = X - mean

6
backend/common/compute/estimate_distribution.py
View File

@@ -2,7 +2,7 @@ import numba
import concurrent.futures
import numpy as np
from scipy import sparse
from backend.common.constants import XApproxDistribution
from backend.common.constants import XApproximateDistribution
@numba.njit(fastmath=True, error_model="numpy", nogil=True)
@@ -29,7 +29,7 @@ def min_max(arr):
return min_val, max_val
def estimate_approximate_distribution(X) -> XApproxDistribution:
def estimate_approximate_distribution(X) -> XApproximateDistribution:
"""
Estimate the distribution (normal, count) of the X matrix.
@@ -59,4 +59,4 @@ def estimate_approximate_distribution(X) -> XApproxDistribution:
min_val, max_val = min_max(Xdata)
excess_range = (max_val - min_val) > 24
return XApproxDistribution.COUNT if excess_range else XApproxDistribution.NORMAL
return XApproximateDistribution.COUNT if excess_range else XApproximateDistribution.NORMAL

2
backend/common/constants.py
View File

@@ -24,7 +24,7 @@ class DiffExpMode(AugmentedEnum):
VAR_FILTER = "varFilter"
class XApproxDistribution(AugmentedEnum):
class XApproximateDistribution(AugmentedEnum):
NORMAL = "normal"
COUNT = "count"

12
backend/czi_hosted/common/config/dataset_config.py
View File

@@ -44,7 +44,7 @@ class DatasetConfig(BaseConfig):
self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
self.diffexp__top_n = default_config["diffexp"]["top_n"]
self.X_approx_distribution = default_config["X_approx_distribution"]
self.X_approximate_distribution = default_config["X_approximate_distribution"]
except KeyError as e:
raise ConfigurationError(f"Unexpected config: {str(e)}")
@@ -60,7 +60,7 @@ class DatasetConfig(BaseConfig):
self.handle_user_annotations(context)
self.handle_embeddings()
self.handle_diffexp(context)
self.handle_X_approx_distribution()
self.handle_X_approximate_distribution()
def handle_app(self):
self.validate_correct_type_of_configuration_attribute("app__scripts", list)
@@ -203,9 +203,9 @@ class DatasetConfig(BaseConfig):
"running differential expression may take longer or fail."
)
def handle_X_approx_distribution(self):
self.validate_correct_type_of_configuration_attribute("X_approx_distribution", str)
if self.X_approx_distribution not in ["normal", "count"]:
def handle_X_approximate_distribution(self):
self.validate_correct_type_of_configuration_attribute("X_approximate_distribution", str)
if self.X_approximate_distribution not in ["normal", "count"]:
raise ConfigurationError(
"X_approx_distribution has unknown value -- must be 'normal' or 'count'."
"X_approximate_distribution has unknown value -- must be 'normal' or 'count'."
)

14
backend/czi_hosted/data_anndata/anndata_adaptor.py
View File

@@ -8,7 +8,7 @@ from scipy import sparse
import backend.common.compute.diffexp_generic as diffexp_generic
from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
from backend.common.constants import Axis, MAX_LAYOUTS, XApproxDistribution
from backend.common.constants import Axis, MAX_LAYOUTS, XApproximateDistribution
from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata
from backend.common.errors import PrepareError, DatasetAccessError, ConfigurationError
from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
@@ -28,7 +28,7 @@ class AnndataAdaptor(DataAdaptor):
def __init__(self, data_locator, app_config=None, dataset_config=None):
super().__init__(data_locator, app_config, dataset_config)
self.data = None
self.X_approx_distribution = None
self.X_approximate_distribution = None
self._load_data(data_locator)
self._validate_and_initialize()
@@ -191,9 +191,9 @@ class AnndataAdaptor(DataAdaptor):
self.gene_count = self.data.shape[1]
self._create_schema()
if self.dataset_config.X_approx_distribution == "auto":
raise ConfigurationError("X-approx-distribution 'auto' mode unsupported.")
self.X_approx_distribution = self.dataset_config.X_approx_distribution
if self.dataset_config.X_approximate_distribution == "auto":
raise ConfigurationError("X-approximate-distribution 'auto' mode unsupported.")
self.X_approximate_distribution = self.dataset_config.X_approximate_distribution
# heuristic
n_values = self.data.shape[0] * self.data.shape[1]
@@ -327,8 +327,8 @@ class AnndataAdaptor(DataAdaptor):
X = self.data.X[obs_mask, var_mask]
return X
def get_X_approx_distribution(self) -> XApproxDistribution:
return self.X_approx_distribution
def get_X_approximate_distribution(self) -> XApproximateDistribution:
return self.X_approximate_distribution
def get_shape(self):
return self.data.shape

4
backend/czi_hosted/data_common/data_adaptor.py
View File

@@ -7,7 +7,7 @@ from scipy import sparse
from server_timing import Timing as ServerTiming
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.common.constants import Axis, XApproxDistribution
from backend.common.constants import Axis, XApproximateDistribution
from backend.common.errors import (
FilterError,
JSONEncodingValueError,
@@ -84,7 +84,7 @@ class DataAdaptor(metaclass=ABCMeta):
pass
@abstractmethod
def get_X_approx_distribution(self) -> XApproxDistribution:
def get_X_approximate_distribution(self) -> XApproximateDistribution:
"""return the approximate distribution of the X matrix."""
pass

14
backend/czi_hosted/data_cxg/cxg_adaptor.py
View File

@@ -8,7 +8,7 @@ import pandas as pd
import tiledb
from server_timing import Timing as ServerTiming
from backend.common.constants import Axis, XApproxDistribution
from backend.common.constants import Axis, XApproximateDistribution
from backend.common.errors import DatasetAccessError, ConfigurationError
from backend.czi_hosted.common.immutable_kvcache import ImmutableKVCache
from backend.common.utils.type_conversion_utils import get_schema_type_hint_from_dtype
@@ -37,7 +37,7 @@ class CxgAdaptor(DataAdaptor):
self.lsuri_results = ImmutableKVCache(lambda key: self._lsuri(uri=key, tiledb_ctx=self.tiledb_ctx))
self.arrays = ImmutableKVCache(lambda key: self._open_array(uri=key, tiledb_ctx=self.tiledb_ctx))
self.schema = None
self.X_approx_distribution = None
self.X_approximate_distribution = None
self._validate_and_initialize()
@@ -176,9 +176,9 @@ class CxgAdaptor(DataAdaptor):
if cxg_version not in ["0.0", "0.1", "0.2.0"]:
raise DatasetAccessError(f"cxg matrix is not valid: {self.url}")
if self.dataset_config.X_approx_distribution == "auto":
raise ConfigurationError("X-approx-distribution 'auto' mode unsupported.")
self.X_approx_distribution = self.dataset_config.X_approx_distribution
if self.dataset_config.X_approximate_distribution == "auto":
raise ConfigurationError("X-approximate-distribution 'auto' mode unsupported.")
self.X_approximate_distribution = self.dataset_config.X_approximate_distribution
self.title = title
self.about = about
@@ -286,8 +286,8 @@ class CxgAdaptor(DataAdaptor):
data = X.multi_index[obs_items, var_items][""]
return data
def get_X_approx_distribution(self) -> XApproxDistribution:
return self.X_approx_distribution
def get_X_approximate_distribution(self) -> XApproximateDistribution:
return self.X_approximate_distribution
def get_shape(self):
X = self.open_array("X")

2
backend/czi_hosted/default_config.py
View File

@@ -203,7 +203,7 @@ dataset:
lfc_cutoff: 0.01
top_n: 10
X_approx_distribution: normal # currently fixed config
X_approximate_distribution: normal # currently fixed config
external:
# You can retrieve configuration parameters from this config file, the environment,

8
backend/server/cli/launch.py
View File

@@ -151,8 +151,8 @@ def dataset_args(func):
help="URL providing more information about the dataset (hint: must be a fully specified absolute URL).",
)
@click.option(
"--X-approx-distribution",
default=DEFAULT_CONFIG.dataset_config.X_approx_distribution,
"--X-approximate-distribution",
default=DEFAULT_CONFIG.dataset_config.X_approximate_distribution,
show_default=True,
type=click.Choice(["auto", "normal", "count"], case_sensitive=False),
help="Specify the approximate distribution of X matrix values. 'auto' will use a heuristic "
@@ -326,7 +326,7 @@ def launch(
disable_diffexp,
config_file,
dump_default_config,
x_approx_distribution,
x_approximate_distribution,
):
"""Launch the cellxgene data viewer.
This web app lets you explore single-cell expression data.
@@ -385,7 +385,7 @@ def launch(
embeddings__names=embedding,
diffexp__enable=not disable_diffexp,
diffexp__lfc_cutoff=diffexp_lfc_cutoff,
X_approx_distribution=x_approx_distribution,
X_approximate_distribution=x_approximate_distribution,
)
diff = cli_config.server_config.changes_from_default()

12
backend/server/common/config/dataset_config.py
View File

@@ -38,7 +38,7 @@ class DatasetConfig(BaseConfig):
self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
self.diffexp__top_n = default_config["diffexp"]["top_n"]
self.X_approx_distribution = default_config["X_approx_distribution"]
self.X_approximate_distribution = default_config["X_approximate_distribution"]
except KeyError as e:
raise ConfigurationError(f"Unexpected config: {str(e)}")
@@ -52,7 +52,7 @@ class DatasetConfig(BaseConfig):
self.handle_user_annotations(context)
self.handle_embeddings()
self.handle_diffexp(context)
self.handle_X_approx_distribution()
self.handle_X_approximate_distribution()
def get_data_adaptor(self):
server_config = self.app_config.server_config
@@ -186,9 +186,9 @@ class DatasetConfig(BaseConfig):
"CAUTION: due to the size of your dataset, " "running differential expression may take longer or fail."
)
def handle_X_approx_distribution(self):
self.validate_correct_type_of_configuration_attribute("X_approx_distribution", str)
if self.X_approx_distribution not in ["auto", "normal", "count"]:
def handle_X_approximate_distribution(self):
self.validate_correct_type_of_configuration_attribute("X_approximate_distribution", str)
if self.X_approximate_distribution not in ["auto", "normal", "count"]:
raise ConfigurationError(
"X_approx_distribution has unknown value -- must be 'auto', 'normal' or 'count'."
"X_approximate_distribution has unknown value -- must be 'auto', 'normal' or 'count'."
)

20
backend/server/data_anndata/anndata_adaptor.py
View File

@@ -9,7 +9,7 @@ from scipy import sparse
import backend.common.compute.diffexp_generic as diffexp_generic
import backend.common.compute.estimate_distribution as estimate_distribution
from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
from backend.common.constants import Axis, MAX_LAYOUTS, XApproxDistribution
from backend.common.constants import Axis, MAX_LAYOUTS, XApproximateDistribution
from backend.server.common.corpora import corpora_get_props_from_anndata
from backend.common.errors import PrepareError, DatasetAccessError
from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
@@ -29,7 +29,7 @@ class AnndataAdaptor(DataAdaptor):
def __init__(self, data_locator, app_config=None, dataset_config=None):
super().__init__(data_locator, app_config, dataset_config)
self.data = None
self.X_approx_distribution = None
self.X_approximate_distribution = None
self._load_data(data_locator)
self._validate_and_initialize()
@@ -192,12 +192,12 @@ class AnndataAdaptor(DataAdaptor):
self.gene_count = self.data.shape[1]
self._create_schema()
if self.dataset_config.X_approx_distribution == "auto":
if self.dataset_config.X_approximate_distribution == "auto":
"""Lazy evaluate the heuristic if we are backed."""
if not self.data.isbacked:
self.X_approx_distribution = estimate_distribution.estimate_approximate_distribution(self.data.X)
self.X_approximate_distribution = estimate_distribution.estimate_approximate_distribution(self.data.X)
else:
self.X_approx_distribution = self.dataset_config.X_approx_distribution
self.X_approximate_distribution = self.dataset_config.X_approximate_distribution
# heuristic
n_values = self.data.shape[0] * self.data.shape[1]
@@ -331,15 +331,15 @@ class AnndataAdaptor(DataAdaptor):
X = self.data.X[obs_mask, var_mask]
return X
def get_X_approx_distribution(self) -> XApproxDistribution:
def get_X_approximate_distribution(self) -> XApproximateDistribution:
"""return the approximate distribution of the X matrix."""
if self.X_approx_distribution is None:
if self.X_approximate_distribution is None:
"""Not yet evaluated."""
assert(self.dataset_config.X_approx_distribution == "auto")
assert(self.dataset_config.X_approximate_distribution == "auto")
self.data = self.data.to_memory() # loads data
self.X_approx_distribution = estimate_distribution.estimate_approximate_distribution(self.data.X)
self.X_approximate_distribution = estimate_distribution.estimate_approximate_distribution(self.data.X)
return self.X_approx_distribution
return self.X_approximate_distribution
def get_shape(self):
return self.data.shape

6
backend/server/data_common/data_adaptor.py
View File

@@ -6,7 +6,7 @@ from scipy import sparse
from server_timing import Timing as ServerTiming
from backend.server.common.config.app_config import AppConfig
from backend.common.constants import Axis, XApproxDistribution
from backend.common.constants import Axis, XApproximateDistribution
from backend.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError, UnsupportedSummaryMethod
from backend.common.utils.utils import jsonify_numpy
from backend.common.fbs.matrix import encode_matrix_fbs
@@ -72,9 +72,9 @@ class DataAdaptor(metaclass=ABCMeta):
the return type is either ndarray or scipy.sparse.spmatrix."""
pass
def get_X_approx_distribution(self) -> XApproxDistribution:
def get_X_approximate_distribution(self) -> XApproximateDistribution:
"""return the approximate distribution of the X matrix."""
return XApproxDistribution.NORMAL
return XApproximateDistribution.NORMAL
@abstractmethod
def get_shape(self):

2
backend/server/default_config.py
View File

@@ -77,7 +77,7 @@ dataset:
lfc_cutoff: 0.01
top_n: 10
X_approx_distribution: auto
X_approximate_distribution: auto
external:
# You can retrieve configuration parameters from this config file, the environment,

2
backend/test/fixtures/czi_hosted_dataset_config_outline.py vendored
View File

@@ -31,5 +31,5 @@ dataset:
lfc_cutoff: {lfc_cutoff}
top_n: {top_n}
X_approx_distribution: {X_approx_distribution}
X_approximate_distribution: {X_approximate_distribution}
"""

2
backend/test/fixtures/dataset_config_outline.py vendored
View File

@@ -28,5 +28,5 @@ dataset:
lfc_cutoff: {lfc_cutoff}
top_n: {top_n}
X_approx_distribution: {X_approx_distribution}
X_approximate_distribution: {X_approximate_distribution}
"""

6
backend/test/test_czi_hosted/unit/common/config/__init__.py
View File

@@ -131,7 +131,7 @@ class ConfigTests(BaseTest):
environment=None,
aws_secrets_manager_region=None,
aws_secrets_manager_secrets=[],
X_approx_distribution="normal",
X_approximate_distribution="normal",
config_file_name="app_config.yml",
):
random_num = random.randrange(999999)
@@ -195,7 +195,7 @@ class ConfigTests(BaseTest):
enable_difexp=enable_difexp,
lfc_cutoff=lfc_cutoff,
top_n=top_n,
X_approx_distribution=X_approx_distribution,
X_approximate_distribution=X_approximate_distribution,
config_file_name=f"temp_dataset_config_{random_num}.yml",
)
external_config = self.custom_external_config(
@@ -231,7 +231,7 @@ class ConfigTests(BaseTest):
enable_difexp="true",
lfc_cutoff=0.01,
top_n=10,
X_approx_distribution="normal",
X_approximate_distribution="normal",
config_file_name="dataset_config.yml",
):
configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)

2
backend/test/test_czi_hosted/unit/compute/test_diffexp_cxg.py
View File

@@ -20,7 +20,7 @@ class DiffExpTest(unittest.TestCase):
adaptor types and different algorithms."""
def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}):
extra_dataset_config["X_approx_distribution"] = "normal" # hardwired for now
extra_dataset_config["X_approximate_distribution"] = "normal" # hardwired for now
config = app_config(path, extra_server_config=extra_server_config, extra_dataset_config=extra_dataset_config)
loader = MatrixDataLoader(path)
adaptor = loader.open(config)

6
backend/test/test_server/unit/common/config/__init__.py
View File

@@ -103,7 +103,7 @@ class ConfigTests(unittest.TestCase):
environment=None,
aws_secrets_manager_region=None,
aws_secrets_manager_secrets=[],
X_approx_distribution="auto",
X_approximate_distribution="auto",
config_file_name="app_config.yml",
):
random_num = random.randrange(999999)
@@ -151,7 +151,7 @@ class ConfigTests(unittest.TestCase):
enable_difexp=enable_difexp,
lfc_cutoff=lfc_cutoff,
top_n=top_n,
X_approx_distribution=X_approx_distribution,
X_approximate_distribution=X_approximate_distribution,
config_file_name=f"temp_dataset_config_{random_num}.yml",
)
external_config = self.custom_external_config(
@@ -187,7 +187,7 @@ class ConfigTests(unittest.TestCase):
enable_difexp="true",
lfc_cutoff=0.01,
top_n=10,
X_approx_distribution="auto",
X_approximate_distribution="auto",
config_file_name="dataset_config.yml",
):
configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)

16
backend/test/test_server/unit/compute/test_est_dist.py
View File

@@ -2,7 +2,7 @@ import unittest
import numpy as np
from scipy import sparse
from backend.common.compute.estimate_distribution import estimate_approximate_distribution
from backend.common.constants import XApproxDistribution
from backend.common.constants import XApproximateDistribution
from backend.server.data_common.matrix_loader import MatrixDataLoader
from backend.test.test_server.unit import app_config
from backend.test import PROJECT_ROOT
@@ -20,22 +20,22 @@ class EstDistTest(unittest.TestCase):
def test_adaptestimate_approximate_distribution(self):
adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
self.assertEqual(adaptor.get_X_approx_distribution(), XApproxDistribution.NORMAL)
self.assertEqual(adaptor.get_X_approximate_distribution(), XApproximateDistribution.NORMAL)
def test_estimate_approximate_distribution(self):
raw = np.random.exponential(scale=1000, size=(100, 40))
# ndarray
self.assertEqual(estimate_approximate_distribution(raw), XApproxDistribution.COUNT)
self.assertEqual(estimate_approximate_distribution(np.log1p(raw)), XApproxDistribution.NORMAL)
self.assertEqual(estimate_approximate_distribution(raw), XApproximateDistribution.COUNT)
self.assertEqual(estimate_approximate_distribution(np.log1p(raw)), XApproximateDistribution.NORMAL)
# csr_matrix
self.assertEqual(estimate_approximate_distribution(sparse.csr_matrix(raw)), XApproxDistribution.COUNT)
self.assertEqual(estimate_approximate_distribution(sparse.csr_matrix(raw)), XApproximateDistribution.COUNT)
self.assertEqual(
estimate_approximate_distribution(sparse.csr_matrix(np.log1p(raw))), XApproxDistribution.NORMAL
estimate_approximate_distribution(sparse.csr_matrix(np.log1p(raw))), XApproximateDistribution.NORMAL
)
# BIG (ie, trigger MT)
big = np.random.exponential(scale=100, size=(1_000_000, 100))
self.assertEqual(estimate_approximate_distribution(big), XApproxDistribution.COUNT)
self.assertEqual(estimate_approximate_distribution(np.log1p(big)), XApproxDistribution.NORMAL)
self.assertEqual(estimate_approximate_distribution(big), XApproximateDistribution.COUNT)
self.assertEqual(estimate_approximate_distribution(np.log1p(big)), XApproximateDistribution.NORMAL)

6
backend/test/test_server/unit/data_anndata/test_anndata_adaptor.py
View File

@@ -22,7 +22,7 @@ Test the anndata adaptor using the pbmc3k data set.
@parameterized_class(
("data_locator", "backed", "X_approx_distribution"),
("data_locator", "backed", "X_approximate_distribution"),
[
(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False, "auto"),
(f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", False, "auto"),
@@ -41,7 +41,9 @@ Test the anndata adaptor using the pbmc3k data set.
class AdaptorTest(unittest.TestCase):
def setUp(self):
config = app_config(
self.data_locator, self.backed, extra_dataset_config=dict(X_approx_distribution=self.X_approx_distribution)
self.data_locator,
self.backed,
extra_dataset_config=dict(X_approximate_distribution=self.X_approximate_distribution),
)
self.data = AnndataAdaptor(DataLocator(self.data_locator), config)