fix for incorrect stats computation in diff exp t-test (#2318)

* 2211 fixes

* lint

* lint

* add missing test and bug found by test

* change terminology for count distribution

* update scanpy requirement

* update scanpy requirement

backend/common/compute/diffexp_generic.py

@@ -1,5 +1,6 @@
 import numpy as np
 from scipy import sparse, stats
+from backend.common.constants import XApproxDistribution
 
 
 def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
@@ -7,7 +8,7 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
     Return differential expression statistics for top N variables.
 
     Algorithm:
-    - compute log fold change (log2(meanA/meanB))
+    - compute fold change
     - compute Welch's t-test statistic and pvalue (w/ Bonferroni correction)
     - return top N abs(logfoldchange) where lfc > diffexp_lfc_cutoff
 
@@ -26,21 +27,24 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
     :param top_n: number of variables to return stats for
     :param diffexp_lfc_cutoff: minimum
     absolute value returning [ varindex, logfoldchange, pval, pval_adj ] for top N genes
-    :return:  for top N genes, {"positive": for top N genes, [ varindex, logfoldchange, pval, pval_adj ], "negative": for top N genes, [ varindex, logfoldchange, pval, pval_adj ]}
+    :return:  for top N genes, {"positive": for top N genes, [ varindex, foldchange, pval, pval_adj ], "negative": for top N genes, [ varindex, foldchange, pval, pval_adj ]}
     """
 
+    X_approx_distribution = adaptor.get_X_approx_distribution()
     dataA = adaptor.get_X_array(maskA, None)
     dataB = adaptor.get_X_array(maskB, None)
 
     # mean, variance, N - calculate for both selections
-    meanA, vA, nA = mean_var_n(dataA)
+    meanA, vA, nA = mean_var_n(dataA, X_approx_distribution)
-    meanB, vB, nB = mean_var_n(dataB)
+    meanB, vB, nB = mean_var_n(dataB, X_approx_distribution)
     res = diffexp_ttest_from_mean_var(meanA, vA, nA, meanB, vB, nB, top_n, diffexp_lfc_cutoff)
 
     return res
 
 
 def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp_lfc_cutoff):
+    # IMPORTANT NOTE: this code assumes the data is normally distributed and/or already logged.
+
     n_var = meanA.shape[0]
     top_n = min(top_n, n_var)
 
@@ -64,15 +68,15 @@ def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp
     pvals_adj = pvals * n_var
     pvals_adj[pvals_adj > 1] = 1  # cap adjusted p-value at 1
 
-    # logfoldchanges: log2(meanA / meanB)
+    # log fold change. The data is normally distributed/logged, so just subtract the means.
-    logfoldchanges = np.log2(np.abs((meanA + 1e-9) / (meanB + 1e-9)))
+    logfoldchanges = meanA - meanB
 
     stats_to_sort = tscores
     # find all with lfc > cutoff
     lfc_above_cutoff_idx = np.nonzero(np.abs(logfoldchanges) > diffexp_lfc_cutoff)[0]
 
     # derive sort order
-    if lfc_above_cutoff_idx.shape[0] > top_n*2:
+    if lfc_above_cutoff_idx.shape[0] > top_n * 2:
         # partition top N
         rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], (top_n, -top_n))
         rel_t_partition_top_n = np.concatenate((rel_t_partition[-top_n:], rel_t_partition[:top_n]))
@@ -95,16 +99,21 @@ def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp
     pvals_adj_top_n = pvals_adj[sort_order]
 
     # varIndex, logfoldchange, pval, pval_adj
-    result = {"positive": [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in
+    result = {
-                           range(top_n)],
+        "positive": [
-              "negative": [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in
+            [sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n)
-                           range(-1, -1 - top_n, -1)], }
+        ],
+        "negative": [
+            [sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]]
+            for i in range(-1, -1 - top_n, -1)
+        ],
+    }
 
     return result
 
 
 # Convenience function which handles sparse data
-def mean_var_n(X):
+def mean_var_n(X, X_approx_distribution=XApproxDistribution.NORMAL):
     """
     Two-pass variance calculation.  Numerically (more) stable
     than naive methods (and same method used by numpy.var())
@@ -122,16 +131,27 @@ def mean_var_n(X):
     with np.errstate(divide="call", invalid="call", call=fp_err_set):
         n = X.shape[0]
         if sparse.issparse(X):
+            if X_approx_distribution == XApproxDistribution.COUNT:
+                X = X.log1p()
             mean = X.mean(axis=0).A1
             dfm = X - mean
             sumsq = np.sum(np.multiply(dfm, dfm), axis=0).A1
             v = sumsq / (n - 1)
         else:
+            if X_approx_distribution == XApproxDistribution.COUNT:
+                X = np.log1p(X)
             mean = X.mean(axis=0)
             dfm = X - mean
             sumsq = np.sum(np.multiply(dfm, dfm), axis=0)
             v = sumsq / (n - 1)
 
+    # AnnData does not guarantee that operations on a view of X will
+    # return an ndarray, so force the cast if it wasn't done for us.
+    if type(mean) is not np.ndarray:
+        mean = mean.toarray()
+    if type(v) is not np.ndarray:
+        v = v.toarray()
+
     if fp_err_occurred:
         mean[np.isfinite(mean) == False] = 0  # noqa: E712
         v[np.isfinite(v) == False] = 0  # noqa: E712

backend/common/compute/estimate_distribution.py

@@ -0,0 +1,62 @@
+import numba
+import concurrent.futures
+import numpy as np
+from scipy import sparse
+from backend.common.constants import XApproxDistribution
+
+
+@numba.njit(fastmath=True, error_model="numpy", nogil=True)
+def min_max(arr):
+    """Return (min, max) values for the ndarray."""
+    n = arr.size
+    odd = n % 2
+    if not odd:
+        n -= 1
+    max_val = min_val = arr[0]
+    i = 1
+    while i < n:
+        x = arr[i]
+        y = arr[i + 1]
+        if x > y:
+            x, y = y, x
+        min_val = min(x, min_val)
+        max_val = max(y, max_val)
+        i += 2
+    if not odd:
+        x = arr[n]
+        min_val = min(x, min_val)
+        max_val = max(x, max_val)
+    return min_val, max_val
+
+
+def estimate_approximate_distribution(X) -> XApproxDistribution:
+    """
+    Estimate the distribution (normal, count) of the X matrix.
+
+    Currently this is based upon the assumption that scRNA-seq data is
+    exponentially distributed in its raw (count) form, and when logged,
+    any (max-min) range in excess of 24 is implies tens of millions of
+    observations of a single feature and so is extremely unlikely.
+    """
+    if sparse.isspmatrix_csc(X) or sparse.isspmatrix_csr(X):
+        Xdata = X.data
+    elif type(X) is np.ndarray:
+        Xdata = X.reshape(
+            X.size,
+        )
+    else:
+        raise TypeError(f"Unsupported matrix type: {str(type(X))}")
+
+    CHUNKSIZE = 1 << 24
+    if Xdata.size > CHUNKSIZE:
+        min_val = max_val = Xdata[0]
+        with concurrent.futures.ThreadPoolExecutor() as tp:
+            for (_min, _max) in tp.map(min_max, [Xdata[i : i + CHUNKSIZE] for i in range(0, Xdata.size, CHUNKSIZE)]):
+                min_val = min(_min, min_val)
+                max_val = max(_max, max_val)
+
+    else:
+        min_val, max_val = min_max(Xdata)
+
+    excess_range = (max_val - min_val) > 24
+    return XApproxDistribution.COUNT if excess_range else XApproxDistribution.NORMAL

backend/common/constants.py

@@ -24,6 +24,11 @@ class DiffExpMode(AugmentedEnum):
     VAR_FILTER = "varFilter"
 
 
+class XApproxDistribution(AugmentedEnum):
+    NORMAL = "normal"
+    COUNT = "count"
+
+
 JSON_NaN_to_num_warning_msg = "JSON encoding failure - please verify all data are finite values (no NaN or Infinities)"
 REACTIVE_LIMIT = 1_000_000
 

backend/czi_hosted/common/config/dataset_config.py

@@ -44,6 +44,8 @@ class DatasetConfig(BaseConfig):
             self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
             self.diffexp__top_n = default_config["diffexp"]["top_n"]
 
+            self.X_approx_distribution = default_config["X_approx_distribution"]
+
         except KeyError as e:
             raise ConfigurationError(f"Unexpected config: {str(e)}")
 
@@ -58,6 +60,7 @@ class DatasetConfig(BaseConfig):
         self.handle_user_annotations(context)
         self.handle_embeddings()
         self.handle_diffexp(context)
+        self.handle_X_approx_distribution()
 
     def handle_app(self):
         self.validate_correct_type_of_configuration_attribute("app__scripts", list)
@@ -199,3 +202,10 @@ class DatasetConfig(BaseConfig):
                         "CAUTION: due to the size of your dataset, "
                         "running differential expression may take longer or fail."
                     )
+
+    def handle_X_approx_distribution(self):
+        self.validate_correct_type_of_configuration_attribute("X_approx_distribution", str)
+        if self.X_approx_distribution not in ["normal", "count"]:
+            raise ConfigurationError(
+                "X_approx_distribution has unknown value -- must be 'normal' or 'count'."
+            )

backend/czi_hosted/data_anndata/anndata_adaptor.py

@@ -8,9 +8,9 @@ from scipy import sparse
 
 import backend.common.compute.diffexp_generic as diffexp_generic
 from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
-from backend.common.constants import Axis, MAX_LAYOUTS
+from backend.common.constants import Axis, MAX_LAYOUTS, XApproxDistribution
 from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata
-from backend.common.errors import PrepareError, DatasetAccessError
+from backend.common.errors import PrepareError, DatasetAccessError, ConfigurationError
 from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
 from backend.czi_hosted.data_common.data_adaptor import DataAdaptor
 from backend.common.fbs.matrix import encode_matrix_fbs
@@ -28,6 +28,7 @@ class AnndataAdaptor(DataAdaptor):
     def __init__(self, data_locator, app_config=None, dataset_config=None):
         super().__init__(data_locator, app_config, dataset_config)
         self.data = None
+        self.X_approx_distribution = None
         self._load_data(data_locator)
         self._validate_and_initialize()
 
@@ -190,6 +191,10 @@ class AnndataAdaptor(DataAdaptor):
         self.gene_count = self.data.shape[1]
         self._create_schema()
 
+        if self.dataset_config.X_approx_distribution == "auto":
+            raise ConfigurationError("X-approx-distribution 'auto' mode unsupported.")
+        self.X_approx_distribution = self.dataset_config.X_approx_distribution
+
         # heuristic
         n_values = self.data.shape[0] * self.data.shape[1]
         if (n_values > 1e8 and self.server_config.adaptor__anndata_adaptor__backed is True) or (n_values > 5e8):
@@ -309,13 +314,22 @@ class AnndataAdaptor(DataAdaptor):
         return convert_anndata_category_colors_to_cxg_category_colors(self.data)
 
     def get_X_array(self, obs_mask=None, var_mask=None):
+        # H5Py does not support boolean indexing (masks), so convert to integer indexing
+        # when backed (ie, when AnnData is using H5Py indexing)
         if obs_mask is None:
             obs_mask = slice(None)
+        elif self.data.isbacked and obs_mask.dtype == bool:
+            obs_mask = obs_mask.nonzero()[0]
         if var_mask is None:
             var_mask = slice(None)
+        elif self.data.isbacked and var_mask.dtype == bool:
+            var_mask = var_mask.nonzero()[0]
         X = self.data.X[obs_mask, var_mask]
         return X
 
+    def get_X_approx_distribution(self) -> XApproxDistribution:
+        return self.X_approx_distribution
+
     def get_shape(self):
         return self.data.shape
 

backend/czi_hosted/data_common/data_adaptor.py

@@ -7,8 +7,14 @@ from scipy import sparse
 from server_timing import Timing as ServerTiming
 
 from backend.czi_hosted.common.config.app_config import AppConfig
-from backend.common.constants import Axis
+from backend.common.constants import Axis, XApproxDistribution
-from backend.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError, UnsupportedSummaryMethod, DatasetAccessError
+from backend.common.errors import (
+    FilterError,
+    JSONEncodingValueError,
+    ExceedsLimitError,
+    UnsupportedSummaryMethod,
+    DatasetAccessError,
+)
 from backend.common.utils.utils import jsonify_numpy
 from backend.common.fbs.matrix import encode_matrix_fbs
 
@@ -77,6 +83,11 @@ class DataAdaptor(metaclass=ABCMeta):
         the return type is either ndarray or scipy.sparse.spmatrix."""
         pass
 
+    @abstractmethod
+    def get_X_approx_distribution(self) -> XApproxDistribution:
+        """return the approximate distribution of the X matrix."""
+        pass
+
     @abstractmethod
     def get_shape(self):
         pass
@@ -158,7 +169,7 @@ class DataAdaptor(metaclass=ABCMeta):
         mask = np.zeros((count,), dtype=np.bool)
         for i in filter:
             if type(i) == list:
-                mask[i[0]: i[1]] = True
+                mask[i[0] : i[1]] = True
             else:
                 mask[i] = True
         return mask
@@ -316,12 +327,13 @@ class DataAdaptor(metaclass=ABCMeta):
             top_n = self.dataset_config.diffexp__top_n
 
         if self.server_config.exceeds_limit(
-                "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
+            "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
         ):
             raise ExceedsLimitError("Diffexp request exceeds max cell count limit")
 
         result = self.compute_diffexp_ttest(
-            maskA=obs_mask_A, maskB=obs_mask_B, top_n=top_n, lfc_cutoff=self.dataset_config.diffexp__lfc_cutoff)
+            maskA=obs_mask_A, maskB=obs_mask_B, top_n=top_n, lfc_cutoff=self.dataset_config.diffexp__lfc_cutoff
+        )
 
         try:
             return jsonify_numpy(result)

backend/czi_hosted/data_cxg/cxg_adaptor.py

@@ -8,7 +8,7 @@ import pandas as pd
 import tiledb
 from server_timing import Timing as ServerTiming
 
-from backend.common.constants import Axis
+from backend.common.constants import Axis, XApproxDistribution
 from backend.common.errors import DatasetAccessError, ConfigurationError
 from backend.czi_hosted.common.immutable_kvcache import ImmutableKVCache
 from backend.common.utils.type_conversion_utils import get_schema_type_hint_from_dtype
@@ -37,6 +37,7 @@ class CxgAdaptor(DataAdaptor):
         self.lsuri_results = ImmutableKVCache(lambda key: self._lsuri(uri=key, tiledb_ctx=self.tiledb_ctx))
         self.arrays = ImmutableKVCache(lambda key: self._open_array(uri=key, tiledb_ctx=self.tiledb_ctx))
         self.schema = None
+        self.X_approx_distribution = None
 
         self._validate_and_initialize()
 
@@ -175,6 +176,10 @@ class CxgAdaptor(DataAdaptor):
         if cxg_version not in ["0.0", "0.1", "0.2.0"]:
             raise DatasetAccessError(f"cxg matrix is not valid: {self.url}")
 
+        if self.dataset_config.X_approx_distribution == "auto":
+            raise ConfigurationError("X-approx-distribution 'auto' mode unsupported.")
+        self.X_approx_distribution = self.dataset_config.X_approx_distribution
+
         self.title = title
         self.about = about
         self.cxg_version = cxg_version
@@ -281,6 +286,9 @@ class CxgAdaptor(DataAdaptor):
                 data = X.multi_index[obs_items, var_items][""]
             return data
 
+    def get_X_approx_distribution(self) -> XApproxDistribution:
+        return self.X_approx_distribution
+
     def get_shape(self):
         X = self.open_array("X")
         return X.shape

backend/czi_hosted/default_config.py

@@ -203,6 +203,8 @@ dataset:
     lfc_cutoff: 0.01
     top_n: 10
 
+  X_approx_distribution: normal # currently fixed config
+
 external:
   # You can retrieve configuration parameters from this config file, the environment,
   # the AWS secrets manager, or from the "cellxgene launch" command line arguments.

backend/czi_hosted/requirements-prepare.txt

@@ -1,4 +1,4 @@
 python-igraph
 louvain>=0.6
-scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here
+scanpy
 umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn

backend/czi_hosted/requirements.txt

@@ -1,4 +1,4 @@
-anndata>=0.7.0
+anndata>=0.7.6 # we use to_memory(), added in 0.7.6
 boto3>=1.12.18
 click>=7.1.2
 Flask>=1.0.2,<2.0.0 # Flask 2.0 is not compatible with the latest version of Flask-RESTful (0.3.8)
@@ -11,7 +11,7 @@ flatbuffers>=1.11.0,<2.0.0 # cellxgene is not compatible with 2.0.0. Requires mi
 flatten-dict>=0.2.0
 fsspec>=0.4.4,<0.8.0
 gunicorn>=20.0.4
-h5py<3.0.0 # h5py>=3.0.0 had a breaking change; there is a fix in anndata>=0.7.5
+h5py>=3.0.0
 numba>=0.49.1,<0.53.0
 numpy>=1.15.0
 packaging>=20.0

backend/server/cli/launch.py

@@ -150,6 +150,14 @@ def dataset_args(func):
         metavar="<URL>",
         help="URL providing more information about the dataset (hint: must be a fully specified absolute URL).",
     )
+    @click.option(
+        "--X-approx-distribution",
+        default=DEFAULT_CONFIG.dataset_config.X_approx_distribution,
+        show_default=True,
+        type=click.Choice(["auto", "normal", "count"], case_sensitive=False),
+        help="Specify the approximate distribution of X matrix values. 'auto' will use a heuristic "
+        "to determine the approximate distribution.  Mode 'auto' is incompatible with --backed.",
+    )
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
         return func(*args, **kwargs)
@@ -318,6 +326,7 @@ def launch(
     disable_diffexp,
     config_file,
     dump_default_config,
+    x_approx_distribution,
 ):
     """Launch the cellxgene data viewer.
     This web app lets you explore single-cell expression data.
@@ -376,6 +385,7 @@ def launch(
             embeddings__names=embedding,
             diffexp__enable=not disable_diffexp,
             diffexp__lfc_cutoff=diffexp_lfc_cutoff,
+            X_approx_distribution=x_approx_distribution,
         )
 
         diff = cli_config.server_config.changes_from_default()

backend/server/common/config/dataset_config.py

@@ -38,6 +38,8 @@ class DatasetConfig(BaseConfig):
             self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
             self.diffexp__top_n = default_config["diffexp"]["top_n"]
 
+            self.X_approx_distribution = default_config["X_approx_distribution"]
+
         except KeyError as e:
             raise ConfigurationError(f"Unexpected config: {str(e)}")
 
@@ -50,6 +52,7 @@ class DatasetConfig(BaseConfig):
         self.handle_user_annotations(context)
         self.handle_embeddings()
         self.handle_diffexp(context)
+        self.handle_X_approx_distribution()
 
     def get_data_adaptor(self):
         server_config = self.app_config.server_config
@@ -182,3 +185,10 @@ class DatasetConfig(BaseConfig):
             context["messagefn"](
                 "CAUTION: due to the size of your dataset, " "running differential expression may take longer or fail."
             )
+
+    def handle_X_approx_distribution(self):
+        self.validate_correct_type_of_configuration_attribute("X_approx_distribution", str)
+        if self.X_approx_distribution not in ["auto", "normal", "count"]:
+            raise ConfigurationError(
+                "X_approx_distribution has unknown value -- must be 'auto', 'normal' or 'count'."
+            )

backend/server/data_anndata/anndata_adaptor.py

@@ -7,8 +7,9 @@ from pandas.core.dtypes.dtypes import CategoricalDtype
 from scipy import sparse
 
 import backend.common.compute.diffexp_generic as diffexp_generic
+import backend.common.compute.estimate_distribution as estimate_distribution
 from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
-from backend.common.constants import Axis, MAX_LAYOUTS
+from backend.common.constants import Axis, MAX_LAYOUTS, XApproxDistribution
 from backend.server.common.corpora import corpora_get_props_from_anndata
 from backend.common.errors import PrepareError, DatasetAccessError
 from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
@@ -28,6 +29,7 @@ class AnndataAdaptor(DataAdaptor):
     def __init__(self, data_locator, app_config=None, dataset_config=None):
         super().__init__(data_locator, app_config, dataset_config)
         self.data = None
+        self.X_approx_distribution = None
         self._load_data(data_locator)
         self._validate_and_initialize()
 
@@ -190,6 +192,13 @@ class AnndataAdaptor(DataAdaptor):
         self.gene_count = self.data.shape[1]
         self._create_schema()
 
+        if self.dataset_config.X_approx_distribution == "auto":
+            """Lazy evaluate the heuristic if we are backed."""
+            if not self.data.isbacked:
+                self.X_approx_distribution = estimate_distribution.estimate_approximate_distribution(self.data.X)
+        else:
+            self.X_approx_distribution = self.dataset_config.X_approx_distribution
+
         # heuristic
         n_values = self.data.shape[0] * self.data.shape[1]
         if (n_values > 1e8 and self.server_config.adaptor__anndata_adaptor__backed is True) or (n_values > 5e8):
@@ -309,13 +318,29 @@ class AnndataAdaptor(DataAdaptor):
         return convert_anndata_category_colors_to_cxg_category_colors(self.data)
 
     def get_X_array(self, obs_mask=None, var_mask=None):
+        # H5Py does not support boolean indexing (masks), so convert to integer indexing
+        # when backed (ie, when AnnData is using H5Py indexing)
         if obs_mask is None:
             obs_mask = slice(None)
+        elif self.data.isbacked and obs_mask.dtype == bool:
+            obs_mask = obs_mask.nonzero()[0]
         if var_mask is None:
             var_mask = slice(None)
+        elif self.data.isbacked and var_mask.dtype == bool:
+            var_mask = var_mask.nonzero()[0]
         X = self.data.X[obs_mask, var_mask]
         return X
 
+    def get_X_approx_distribution(self) -> XApproxDistribution:
+        """return the approximate distribution of the X matrix."""
+        if self.X_approx_distribution is None:
+            """Not yet evaluated."""
+            assert(self.dataset_config.X_approx_distribution == "auto")
+            self.data = self.data.to_memory()  # loads data
+            self.X_approx_distribution = estimate_distribution.estimate_approximate_distribution(self.data.X)
+
+        return self.X_approx_distribution
+
     def get_shape(self):
         return self.data.shape
 

backend/server/data_common/data_adaptor.py

@@ -6,7 +6,7 @@ from scipy import sparse
 from server_timing import Timing as ServerTiming
 
 from backend.server.common.config.app_config import AppConfig
-from backend.common.constants import Axis
+from backend.common.constants import Axis, XApproxDistribution
 from backend.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError, UnsupportedSummaryMethod
 from backend.common.utils.utils import jsonify_numpy
 from backend.common.fbs.matrix import encode_matrix_fbs
@@ -72,6 +72,10 @@ class DataAdaptor(metaclass=ABCMeta):
         the return type is either ndarray or scipy.sparse.spmatrix."""
         pass
 
+    def get_X_approx_distribution(self) -> XApproxDistribution:
+        """return the approximate distribution of the X matrix."""
+        return XApproxDistribution.NORMAL
+
     @abstractmethod
     def get_shape(self):
         pass

backend/server/default_config.py

@@ -77,6 +77,8 @@ dataset:
     lfc_cutoff: 0.01
     top_n: 10
 
+  X_approx_distribution: auto
+
 external:
   # You can retrieve configuration parameters from this config file, the environment,
   # the AWS secrets manager, or from the "cellxgene launch" command line arguments.

backend/server/requirements-prepare.txt

@@ -1,4 +1,4 @@
 python-igraph>=0.8
 louvain>=0.6
-scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here
+scanpy
 umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn

backend/server/requirements.txt

@@ -1,4 +1,4 @@
-anndata>=0.7.0
+anndata>=0.7.6 # we need to_memory(), added in 0.7.6
 boto3>=1.12.18
 click>=7.1.2
 Flask>=1.0.2,<2.0.0 # Flask 2.0 is not compatible with the latest version of Flask-RESTful (0.3.8)
@@ -11,9 +11,9 @@ flatbuffers>=1.11.0,<2.0.0 # cellxgene is not compatible with 2.0.0. Requires mi
 flatten-dict>=0.2.0
 fsspec>=0.4.4,<0.8.0
 gunicorn>=20.0.4
-h5py<3.0.0 # h5py>=3.0.0 had a breaking change; there is a fix in anndata>=0.7.5
+h5py>=3.0.0
 jinja2>=2.11.3  # Flask sub-dependency. Added due to CVE-2020-28493
-numba>=0.51.2,<0.53.0
+numba>=0.51.2
 numpy>=1.17.5
 packaging>=20.0
 pandas>=1.0,!=1.1  # pandas 1.1 breaks tests, https://github.com/pandas-dev/pandas/issues/35446

backend/test/fixtures/czi_hosted_dataset_config_outline.py

@@ -30,4 +30,6 @@ dataset:
     enable: {enable_difexp}
     lfc_cutoff: {lfc_cutoff}
     top_n: {top_n}
+
+  X_approx_distribution: {X_approx_distribution}
 """

backend/test/fixtures/dataset_config_outline.py

@@ -27,4 +27,6 @@ dataset:
     enable: {enable_difexp}
     lfc_cutoff: {lfc_cutoff}
     top_n: {top_n}
+
+  X_approx_distribution: {X_approx_distribution}
 """

backend/test/test_common/unit/utils/test_type_conversion_utils.py

@@ -70,21 +70,21 @@ class TestTypeConversionUtils(unittest.TestCase):
         self.assertFalse(can_cast)
 
     def test__can_cast_to_int32__int64_is_true(self):
-        array_to_convert = Series(data=["1", "2", "3"], dtype=np.dtype(np.int64))
+        array_to_convert = Series(data=[1, 2, 3], dtype=np.dtype(np.int64))
 
         can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
 
         self.assertTrue(can_cast)
 
     def test__can_cast_to_int32__int16_is_true(self):
-        array_to_convert = Series(data=["1", "2", "3"], dtype=np.dtype(np.int16))
+        array_to_convert = Series(data=[1, 2, 3], dtype=np.dtype(np.int16))
 
         can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
 
         self.assertTrue(can_cast)
 
     def test__can_cast_to_int32__int64_with_large_value_is_false(self):
-        array_to_convert = Series(data=["3000000000", "2", "3"], dtype=np.dtype(np.int64))
+        array_to_convert = Series(data=[3000000000, 2, 3], dtype=np.dtype(np.int64))
 
         can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
 

backend/test/test_czi_hosted/unit/common/config/__init__.py

@@ -131,6 +131,7 @@ class ConfigTests(BaseTest):
         environment=None,
         aws_secrets_manager_region=None,
         aws_secrets_manager_secrets=[],
+        X_approx_distribution="normal",
         config_file_name="app_config.yml",
     ):
         random_num = random.randrange(999999)
@@ -194,6 +195,7 @@ class ConfigTests(BaseTest):
             enable_difexp=enable_difexp,
             lfc_cutoff=lfc_cutoff,
             top_n=top_n,
+            X_approx_distribution=X_approx_distribution,
             config_file_name=f"temp_dataset_config_{random_num}.yml",
         )
         external_config = self.custom_external_config(
@@ -229,6 +231,7 @@ class ConfigTests(BaseTest):
         enable_difexp="true",
         lfc_cutoff=0.01,
         top_n=10,
+        X_approx_distribution="normal",
         config_file_name="dataset_config.yml",
     ):
         configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)

backend/test/test_czi_hosted/unit/common/config/test_dataset_config.py

@@ -49,7 +49,7 @@ class TestDatasetConfig(ConfigTests):
     def test_complete_config_checks_all_attr(self, mock_check_attrs):
         mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
         self.dataset_config.complete_config(self.context)
-        self.assertEqual(mock_check_attrs.call_count, 18)
+        self.assertEqual(mock_check_attrs.call_count, 19)
 
     def test_app_sets_script_vars(self):
         config = self.get_config(scripts=["path/to/script"])

backend/test/test_czi_hosted/unit/compute/test_diffexp_cxg.py

@@ -20,6 +20,7 @@ class DiffExpTest(unittest.TestCase):
     adaptor types and different algorithms."""
 
     def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}):
+        extra_dataset_config["X_approx_distribution"] = "normal"  # hardwired for now
         config = app_config(path, extra_server_config=extra_server_config, extra_dataset_config=extra_dataset_config)
         loader = MatrixDataLoader(path)
         adaptor = loader.open(config)
@@ -46,28 +47,28 @@ class DiffExpTest(unittest.TestCase):
         """Checks the results for a specific set of rows selections"""
 
         positive_expects = [
-            [1712, -0.5525154, 0.0051788902660723345, 1.0],
+            [1712, 0.24104056, 0.0051788902660723345, 1.0],
-            [1575, 1.0317602, 0.007830310753043345, 1.0],
+            [1575, 0.2615018, 0.007830310753043345, 1.0],
-            [693, 0.4703904, 0.008715846769131548, 1.0],
+            [693, 0.23106655, 0.008715846769131548, 1.0],
-            [916, 0.9567287, 0.009080596532247588, 1.0],
+            [916, 0.2395215, 0.009080596532247588, 1.0],
-            [77, 0.02665649, 0.010070392939027756, 1.0],
+            [77, 0.22927025, 0.010070392939027756, 1.0],
-            [782, -1.0981874, 0.010161745218916036, 1.0],
+            [782, 0.20581803, 0.010161745218916036, 1.0],
-            [913, 0.5683986, 0.010782030711612685, 1.0],
+            [913, 0.23841085, 0.010782030711612685, 1.0],
-            [910, 0.83164597, 0.014596411069229197, 1.0],
+            [910, 0.21493295, 0.014596411069229197, 1.0],
-            [1727, 0.4127781, 0.015168372104237176, 1.0],
+            [1727, 0.21911663, 0.015168372104237176, 1.0],
-            [1443, -0.8241895, 0.015337080567465522, 1.0]
+            [1443, 0.19814226, 0.015337080567465522, 1.0],
         ]
         negative_expects = [
-            [956, 0.016060986, 0.0008649321884808977, 1.0],
+            [956, -0.29662406, 0.0008649321884808977, 1.0],
-            [1124, 0.96602094, 0.0011717216548271284, 1.0],
+            [1124, -0.2607333, 0.0011717216548271284, 1.0],
-            [1809, 1.1110606, 0.0019304405196777848, 1.0],
+            [1809, -0.24854594, 0.0019304405196777848, 1.0],
-            [1754, 0.5201581, 0.005691734062127954, 1.0],
+            [1754, -0.24683577, 0.005691734062127954, 1.0],
-            [948, 1.6390722, 0.006622111055981219, 1.0],
+            [948, -0.18708363, 0.006622111055981219, 1.0],
-            [1810, 0.78618884, 0.007055917428377063, 1.0],
+            [1810, -0.2172082, 0.007055917428377063, 1.0],
-            [779, 1.5241305, 0.007202934422407284, 1.0],
+            [779, -0.21150622, 0.007202934422407284, 1.0],
-            [576, 0.97873515, 0.008272092578813124, 1.0],
+            [576, -0.19008157, 0.008272092578813124, 1.0],
-            [538, 0.89114505, 0.01062259019889307, 1.0],
+            [538, -0.21803819, 0.01062259019889307, 1.0],
-            [436, 0.3119122, 0.01127515110543434, 1.0]
+            [436, -0.2100364, 0.01127515110543434, 1.0],
         ]
 
         self.compare_diffexp_results(results['positive'], positive_expects)

backend/test/test_server/unit/common/config/__init__.py

@@ -103,6 +103,7 @@ class ConfigTests(unittest.TestCase):
         environment=None,
         aws_secrets_manager_region=None,
         aws_secrets_manager_secrets=[],
+        X_approx_distribution="auto",
         config_file_name="app_config.yml",
     ):
         random_num = random.randrange(999999)
@@ -150,6 +151,7 @@ class ConfigTests(unittest.TestCase):
             enable_difexp=enable_difexp,
             lfc_cutoff=lfc_cutoff,
             top_n=top_n,
+            X_approx_distribution=X_approx_distribution,
             config_file_name=f"temp_dataset_config_{random_num}.yml",
         )
         external_config = self.custom_external_config(
@@ -185,6 +187,7 @@ class ConfigTests(unittest.TestCase):
         enable_difexp="true",
         lfc_cutoff=0.01,
         top_n=10,
+        X_approx_distribution="auto",
         config_file_name="dataset_config.yml",
     ):
         configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)

backend/test/test_server/unit/common/config/test_dataset_config.py

@@ -7,7 +7,7 @@ from unittest.mock import patch
 from backend.server.common.annotations.local_file_csv import AnnotationsLocalFile
 from backend.server.common.config.app_config import AppConfig
 from backend.server.common.config.base_config import BaseConfig
-from backend.test import FIXTURES_ROOT, H5AD_FIXTURE
+from backend.test import H5AD_FIXTURE
 
 from backend.common.errors import ConfigurationError
 from backend.test.test_server.unit.common.config import ConfigTests
@@ -46,7 +46,7 @@ class TestDatasetConfig(ConfigTests):
         mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
         self.dataset_config.complete_config(self.context)
         self.assertIsNotNone(self.config.server_config.data_adaptor)
-        self.assertEqual(mock_check_attrs.call_count, 16)
+        self.assertEqual(mock_check_attrs.call_count, 17)
 
     def test_app_sets_script_vars(self):
         config = self.get_config(scripts=["path/to/script"])

backend/test/test_server/unit/compute/test_diffexp_h5ad.py

@@ -38,28 +38,28 @@ class DiffExpTest(unittest.TestCase):
         """Checks the results for a specific set of rows selections"""
 
         positive_expects = [
-            [1712, -0.5525154, 0.0051788902660723345, 1.0],
+            [1712, 0.24104056, 0.0051788902660723345, 1.0],
-            [1575, 1.0317602, 0.007830310753043345, 1.0],
+            [1575, 0.2615018, 0.007830310753043345, 1.0],
-            [693, 0.4703904, 0.008715846769131548, 1.0],
+            [693, 0.23106655, 0.008715846769131548, 1.0],
-            [916, 0.9567287, 0.009080596532247588, 1.0],
+            [916, 0.2395215, 0.009080596532247588, 1.0],
-            [77, 0.02665649, 0.010070392939027756, 1.0],
+            [77, 0.22927025, 0.010070392939027756, 1.0],
-            [782, -1.0981874, 0.010161745218916036, 1.0],
+            [782, 0.20581803, 0.010161745218916036, 1.0],
-            [913, 0.5683986, 0.010782030711612685, 1.0],
+            [913, 0.23841085, 0.010782030711612685, 1.0],
-            [910, 0.83164597, 0.014596411069229197, 1.0],
+            [910, 0.21493295, 0.014596411069229197, 1.0],
-            [1727, 0.4127781, 0.015168372104237176, 1.0],
+            [1727, 0.21911663, 0.015168372104237176, 1.0],
-            [1443, -0.8241895, 0.015337080567465522, 1.0]
+            [1443, 0.19814226, 0.015337080567465522, 1.0],
         ]
         negative_expects = [
-            [956, 0.016060986, 0.0008649321884808977, 1.0],
+            [956, -0.29662406, 0.0008649321884808977, 1.0],
-            [1124, 0.96602094, 0.0011717216548271284, 1.0],
+            [1124, -0.2607333, 0.0011717216548271284, 1.0],
-            [1809, 1.1110606, 0.0019304405196777848, 1.0],
+            [1809, -0.24854594, 0.0019304405196777848, 1.0],
-            [1754, 0.5201581, 0.005691734062127954, 1.0],
+            [1754, -0.24683577, 0.005691734062127954, 1.0],
-            [948, 1.6390722, 0.006622111055981219, 1.0],
+            [948, -0.18708363, 0.006622111055981219, 1.0],
-            [1810, 0.78618884, 0.007055917428377063, 1.0],
+            [1810, -0.2172082, 0.007055917428377063, 1.0],
-            [779, 1.5241305, 0.007202934422407284, 1.0],
+            [779, -0.21150622, 0.007202934422407284, 1.0],
-            [576, 0.97873515, 0.008272092578813124, 1.0],
+            [576, -0.19008157, 0.008272092578813124, 1.0],
-            [538, 0.89114505, 0.01062259019889307, 1.0],
+            [538, -0.21803819, 0.01062259019889307, 1.0],
-            [436, 0.3119122, 0.01127515110543434, 1.0]
+            [436, -0.2100364, 0.01127515110543434, 1.0],
         ]
 
         self.compare_diffexp_results(results["positive"], positive_expects)

backend/test/test_server/unit/compute/test_est_dist.py

@@ -0,0 +1,41 @@
+import unittest
+import numpy as np
+from scipy import sparse
+from backend.common.compute.estimate_distribution import estimate_approximate_distribution
+from backend.common.constants import XApproxDistribution
+from backend.server.data_common.matrix_loader import MatrixDataLoader
+from backend.test.test_server.unit import app_config
+from backend.test import PROJECT_ROOT
+
+
+class EstDistTest(unittest.TestCase):
+    """Tests the diffexp returns the expected results for one test case, using the h5ad
+    adaptor types and different algorithms."""
+
+    def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}):
+        config = app_config(path, extra_server_config=extra_server_config, extra_dataset_config=extra_dataset_config)
+        loader = MatrixDataLoader(path)
+        adaptor = loader.open(config)
+        return adaptor
+
+    def test_adaptestimate_approximate_distribution(self):
+        adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
+        self.assertEqual(adaptor.get_X_approx_distribution(), XApproxDistribution.NORMAL)
+
+    def test_estimate_approximate_distribution(self):
+        raw = np.random.exponential(scale=1000, size=(100, 40))
+
+        # ndarray
+        self.assertEqual(estimate_approximate_distribution(raw), XApproxDistribution.COUNT)
+        self.assertEqual(estimate_approximate_distribution(np.log1p(raw)), XApproxDistribution.NORMAL)
+
+        # csr_matrix
+        self.assertEqual(estimate_approximate_distribution(sparse.csr_matrix(raw)), XApproxDistribution.COUNT)
+        self.assertEqual(
+            estimate_approximate_distribution(sparse.csr_matrix(np.log1p(raw))), XApproxDistribution.NORMAL
+        )
+
+        # BIG (ie, trigger MT)
+        big = np.random.exponential(scale=100, size=(1_000_000, 100))
+        self.assertEqual(estimate_approximate_distribution(big), XApproxDistribution.COUNT)
+        self.assertEqual(estimate_approximate_distribution(np.log1p(big)), XApproxDistribution.NORMAL)

backend/test/test_server/unit/data_anndata/test_anndata_adaptor.py

@@ -22,19 +22,27 @@ Test the anndata adaptor using the pbmc3k data set.
 
 
 @parameterized_class(
-    ("data_locator", "backed"),
+    ("data_locator", "backed", "X_approx_distribution"),
     [
-        (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False),
+        (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False, "auto"),
-        (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", False),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", False, "auto"),
-        (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", False),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", False, "auto"),
-        (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", True),
+        (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", True, "auto"),
-        (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", True),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", True, "auto"),
-        (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", True),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", True, "auto"),
+        (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False, "normal"),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", False, "normal"),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", False, "normal"),
+        (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", True, "normal"),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", True, "normal"),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", True, "normal"),
     ],
 )
 class AdaptorTest(unittest.TestCase):
     def setUp(self):
-        config = app_config(self.data_locator, self.backed)
+        config = app_config(
+            self.data_locator, self.backed, extra_dataset_config=dict(X_approx_distribution=self.X_approx_distribution)
+        )
         self.data = AnndataAdaptor(DataLocator(self.data_locator), config)
 
     def test_init(self):
@@ -90,7 +98,8 @@ class AdaptorTest(unittest.TestCase):
 
     def test_schema_produces_error(self):
         self.data.data.obs["time"] = pd.Series(
-            list([time.time() for i in range(self.data.cell_count)]), dtype="datetime64[ns]",
+            list([time.time() for i in range(self.data.cell_count)]),
+            dtype="datetime64[ns]",
         )
         with pytest.raises(TypeError):
             self.data._create_schema()
@@ -107,7 +116,7 @@ class AdaptorTest(unittest.TestCase):
         self.assertTrue((Y >= 0).all() and (Y <= 1).all())
 
     def test_layout_fields(self):
-        """ X_pca, X_tsne, X_umap are available """
+        """X_pca, X_tsne, X_umap are available"""
         fbs = self.data.layout_to_fbs_matrix(["pca"])
         layout = decode_fbs.decode_matrix_FBS(fbs)
         self.assertEqual(layout["n_cols"], 2)
@@ -127,7 +136,8 @@ class AdaptorTest(unittest.TestCase):
         self.assertEqual(annotations["n_cols"], 5)
         obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"]
         self.assertEqual(
-            annotations["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
+            annotations["col_idx"],
+            [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
         )
 
         fbs = self.data.annotation_to_fbs_matrix("var")
@@ -153,12 +163,12 @@ class AdaptorTest(unittest.TestCase):
         f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
         f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
         result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
-        self.assertEqual(len(result['positive']), 10)
+        self.assertEqual(len(result["positive"]), 10)
-        self.assertEqual(len(result['negative']), 10)
+        self.assertEqual(len(result["negative"]), 10)
 
         result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
-        self.assertEqual(len(result['positive']), 20)
+        self.assertEqual(len(result["positive"]), 20)
-        self.assertEqual(len(result['negative']), 20)
+        self.assertEqual(len(result["negative"]), 20)
 
     def test_data_frame(self):
         f1 = {"var": {"index": [[0, 10]]}}