fix for incorrect stats computation in diff exp t-test (#2318)

* 2211 fixes

* lint

* lint

* add missing test and bug found by test

* change terminology for count distribution

* update scanpy requirement

* update scanpy requirement

backend/test/test_server/unit/compute/test_diffexp_h5ad.py

@@ -38,28 +38,28 @@ class DiffExpTest(unittest.TestCase):
         """Checks the results for a specific set of rows selections"""
 
         positive_expects = [
-            [1712, -0.5525154, 0.0051788902660723345, 1.0],
-            [1575, 1.0317602, 0.007830310753043345, 1.0],
-            [693, 0.4703904, 0.008715846769131548, 1.0],
-            [916, 0.9567287, 0.009080596532247588, 1.0],
-            [77, 0.02665649, 0.010070392939027756, 1.0],
-            [782, -1.0981874, 0.010161745218916036, 1.0],
-            [913, 0.5683986, 0.010782030711612685, 1.0],
-            [910, 0.83164597, 0.014596411069229197, 1.0],
-            [1727, 0.4127781, 0.015168372104237176, 1.0],
-            [1443, -0.8241895, 0.015337080567465522, 1.0]
+            [1712, 0.24104056, 0.0051788902660723345, 1.0],
+            [1575, 0.2615018, 0.007830310753043345, 1.0],
+            [693, 0.23106655, 0.008715846769131548, 1.0],
+            [916, 0.2395215, 0.009080596532247588, 1.0],
+            [77, 0.22927025, 0.010070392939027756, 1.0],
+            [782, 0.20581803, 0.010161745218916036, 1.0],
+            [913, 0.23841085, 0.010782030711612685, 1.0],
+            [910, 0.21493295, 0.014596411069229197, 1.0],
+            [1727, 0.21911663, 0.015168372104237176, 1.0],
+            [1443, 0.19814226, 0.015337080567465522, 1.0],
         ]
         negative_expects = [
-            [956, 0.016060986, 0.0008649321884808977, 1.0],
-            [1124, 0.96602094, 0.0011717216548271284, 1.0],
-            [1809, 1.1110606, 0.0019304405196777848, 1.0],
-            [1754, 0.5201581, 0.005691734062127954, 1.0],
-            [948, 1.6390722, 0.006622111055981219, 1.0],
-            [1810, 0.78618884, 0.007055917428377063, 1.0],
-            [779, 1.5241305, 0.007202934422407284, 1.0],
-            [576, 0.97873515, 0.008272092578813124, 1.0],
-            [538, 0.89114505, 0.01062259019889307, 1.0],
-            [436, 0.3119122, 0.01127515110543434, 1.0]
+            [956, -0.29662406, 0.0008649321884808977, 1.0],
+            [1124, -0.2607333, 0.0011717216548271284, 1.0],
+            [1809, -0.24854594, 0.0019304405196777848, 1.0],
+            [1754, -0.24683577, 0.005691734062127954, 1.0],
+            [948, -0.18708363, 0.006622111055981219, 1.0],
+            [1810, -0.2172082, 0.007055917428377063, 1.0],
+            [779, -0.21150622, 0.007202934422407284, 1.0],
+            [576, -0.19008157, 0.008272092578813124, 1.0],
+            [538, -0.21803819, 0.01062259019889307, 1.0],
+            [436, -0.2100364, 0.01127515110543434, 1.0],
         ]
 
         self.compare_diffexp_results(results["positive"], positive_expects)

backend/test/test_server/unit/compute/test_est_dist.py

@@ -0,0 +1,41 @@
+import unittest
+import numpy as np
+from scipy import sparse
+from backend.common.compute.estimate_distribution import estimate_approximate_distribution
+from backend.common.constants import XApproxDistribution
+from backend.server.data_common.matrix_loader import MatrixDataLoader
+from backend.test.test_server.unit import app_config
+from backend.test import PROJECT_ROOT
+
+
+class EstDistTest(unittest.TestCase):
+    """Tests the diffexp returns the expected results for one test case, using the h5ad
+    adaptor types and different algorithms."""
+
+    def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}):
+        config = app_config(path, extra_server_config=extra_server_config, extra_dataset_config=extra_dataset_config)
+        loader = MatrixDataLoader(path)
+        adaptor = loader.open(config)
+        return adaptor
+
+    def test_adaptestimate_approximate_distribution(self):
+        adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
+        self.assertEqual(adaptor.get_X_approx_distribution(), XApproxDistribution.NORMAL)
+
+    def test_estimate_approximate_distribution(self):
+        raw = np.random.exponential(scale=1000, size=(100, 40))
+
+        # ndarray
+        self.assertEqual(estimate_approximate_distribution(raw), XApproxDistribution.COUNT)
+        self.assertEqual(estimate_approximate_distribution(np.log1p(raw)), XApproxDistribution.NORMAL)
+
+        # csr_matrix
+        self.assertEqual(estimate_approximate_distribution(sparse.csr_matrix(raw)), XApproxDistribution.COUNT)
+        self.assertEqual(
+            estimate_approximate_distribution(sparse.csr_matrix(np.log1p(raw))), XApproxDistribution.NORMAL
+        )
+
+        # BIG (ie, trigger MT)
+        big = np.random.exponential(scale=100, size=(1_000_000, 100))
+        self.assertEqual(estimate_approximate_distribution(big), XApproxDistribution.COUNT)
+        self.assertEqual(estimate_approximate_distribution(np.log1p(big)), XApproxDistribution.NORMAL)