feat: diffexp returns two genesets (#2230)

* feat: return two lists for diffexp (#2221)

* sp

* split out derive sort order, tests passing

* sp

* return diff exp results in two lists

* update

* copy implementation over to desktop

* add tests for two lists

* small fixes to complete backend implementation

* accept new diffexp response

* map diff exp response to genesets

* delete )

* name diffexp genesets with population names

* take constants out of state and allow width prop to override

* shorten mini-histo properly truncate and resize depending on expansion

* prepend new genesets

* rename data within diffexp action

* backend

* move diffexp ttest to common code module, update tests

* update for unit tests

* reference actual var

Co-authored-by: Madison Dunitz <madison.dunitz@chanzuckerberg.com>
Co-authored-by: Madison Dunitz <dunitzm@gmail.com>

backend/test/test_server/performance/run_diffexp.py

@@ -4,7 +4,7 @@ import random
 import time
 import numpy as np
 
-import backend.server.compute.diffexp_generic as diffexp_generic
+import backend.common.compute.diffexp_generic as diffexp_generic
 
 from backend.server.common.config.app_config import AppConfig
 from backend.server.data_common.matrix_loader import MatrixDataLoader

backend/test/test_server/unit/common/test_api.py

@@ -414,7 +414,8 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
         self.assertEqual(result.status_code, HTTPStatus.OK)
         self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data = result.json()
-        self.assertEqual(len(result_data), 7)
+        self.assertEqual(len(result_data['positive']), 7)
+        self.assertEqual(len(result_data['negative']), 7)
 
     def test_diff_exp_indices(self):
         endpoint = "diffexp/obs"
@@ -429,7 +430,8 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
         self.assertEqual(result.status_code, HTTPStatus.OK)
         self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data = result.json()
-        self.assertEqual(len(result_data), 10)
+        self.assertEqual(len(result_data['positive']), 10)
+        self.assertEqual(len(result_data['negative']), 10)
 
     def test_get_summaryvar(self):
         index_col_name = self.schema["schema"]["annotations"]["var"]["index"]

backend/test/test_server/unit/compute/test_diffexp_h5ad.py

@@ -2,13 +2,14 @@ import unittest
 
 import numpy as np
 
+from backend.common.compute import diffexp_generic
 from backend.server.data_common.matrix_loader import MatrixDataLoader
 from backend.test.test_server.unit import app_config
 from backend.test import PROJECT_ROOT
 
 
 class DiffExpTest(unittest.TestCase):
-    """Tests the diffexp returns the expected results for one test case, using different
+    """Tests the diffexp returns the expected results for one test case, using the h5ad
     adaptor types and different algorithms."""
 
     def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}):
@@ -35,19 +36,34 @@ class DiffExpTest(unittest.TestCase):
 
     def check_1_10_2_10(self, results):
         """Checks the results for a specific set of rows selections"""
-        expects = [
+
+        positive_expects = [
+            [1712, -0.5525154, 0.0051788902660723345, 1.0],
+            [1575, 1.0317602, 0.007830310753043345, 1.0],
+            [693, 0.4703904, 0.008715846769131548, 1.0],
+            [916, 0.9567287, 0.009080596532247588, 1.0],
+            [77, 0.02665649, 0.010070392939027756, 1.0],
+            [782, -1.0981874, 0.010161745218916036, 1.0],
+            [913, 0.5683986, 0.010782030711612685, 1.0],
+            [910, 0.83164597, 0.014596411069229197, 1.0],
+            [1727, 0.4127781, 0.015168372104237176, 1.0],
+            [1443, -0.8241895, 0.015337080567465522, 1.0]
+        ]
+        negative_expects = [
             [956, 0.016060986, 0.0008649321884808977, 1.0],
             [1124, 0.96602094, 0.0011717216548271284, 1.0],
             [1809, 1.1110606, 0.0019304405196777848, 1.0],
-            [1712, -0.5525154, 0.0051788902660723345, 1.0],
             [1754, 0.5201581, 0.005691734062127954, 1.0],
             [948, 1.6390722, 0.006622111055981219, 1.0],
             [1810, 0.78618884, 0.007055917428377063, 1.0],
             [779, 1.5241305, 0.007202934422407284, 1.0],
-            [1575, 1.0317602, 0.007830310753043345, 1.0],
             [576, 0.97873515, 0.008272092578813124, 1.0],
+            [538, 0.89114505, 0.01062259019889307, 1.0],
+            [436, 0.3119122, 0.01127515110543434, 1.0]
         ]
-        self.compare_diffexp_results(results, expects)
+
+        self.compare_diffexp_results(results["positive"], positive_expects)
+        self.compare_diffexp_results(results["negative"], negative_expects)
 
     def get_X_col(self, adaptor, cols):
         varmask = np.zeros(adaptor.get_shape()[1], dtype=bool)
@@ -61,3 +77,29 @@ class DiffExpTest(unittest.TestCase):
         maskB = self.get_mask(adaptor, 2, 10)
         results = adaptor.compute_diffexp_ttest(maskA, maskB, 10)
         self.check_1_10_2_10(results)
+
+
+def test_h5ad_default(self):
+    """Test a h5ad adaptor with its default diffexp algorithm (diffexp_cxg)"""
+    adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
+    maskA = self.get_mask(adaptor, 1, 10)
+    maskB = self.get_mask(adaptor, 2, 10)
+
+    # run it through the adaptor
+    results = adaptor.compute_diffexp_ttest(maskA, maskB, 10)
+    self.check_1_10_2_10(results)
+
+    # run it directly
+
+    results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB, 10)
+    self.check_1_10_2_10(results)
+
+
+def test_h5ad_generic(self):
+    """Test a h5ad adaptor with the generic adaptor"""
+    adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
+    maskA = self.get_mask(adaptor, 1, 10)
+    maskB = self.get_mask(adaptor, 2, 10)
+    # run it directly
+    results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB, 10)
+    self.check_1_10_2_10(results)

backend/test/test_server/unit/data_anndata/test_anndata_adaptor.py

@@ -153,9 +153,12 @@ class AdaptorTest(unittest.TestCase):
         f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
         f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
         result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
-        self.assertEqual(len(result), 10)
+        self.assertEqual(len(result['positive']), 10)
+        self.assertEqual(len(result['negative']), 10)
+
         result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
-        self.assertEqual(len(result), 20)
+        self.assertEqual(len(result['positive']), 20)
+        self.assertEqual(len(result['negative']), 20)
 
     def test_data_frame(self):
         f1 = {"var": {"index": [[0, 10]]}}

backend/test/test_server/unit/data_anndata/test_anndata_adaptor_data_load.py

@@ -31,9 +31,11 @@ class DataLoadAdaptorTest(unittest.TestCase):
         f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
         f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
         result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
-        self.assertEqual(len(result), 10)
+        self.assertEqual(len(result['positive']), 10)
+        self.assertEqual(len(result['negative']), 10)
         result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
-        self.assertEqual(len(result), 20)
+        self.assertEqual(len(result['positive']), 20)
+        self.assertEqual(len(result['negative']), 20)
 
 
 class DataLocatorAdaptorTest(unittest.TestCase):