feat: diffexp returns two genesets (#2230)

* feat: return two lists for diffexp (#2221)

* sp

* split out derive sort order, tests passing

* sp

* return diff exp results in two lists

* update

* copy implementation over to desktop

* add tests for two lists

* small fixes to complete backend implementation

* accept new diffexp response

* map diff exp response to genesets

* delete )

* name diffexp genesets with population names

* take constants out of state and allow width prop to override

* shorten mini-histo properly truncate and resize depending on expansion

* prepend new genesets

* rename data within diffexp action

* backend

* move diffexp ttest to common code module, update tests

* update for unit tests

* reference actual var

Co-authored-by: Madison Dunitz <madison.dunitz@chanzuckerberg.com>
Co-authored-by: Madison Dunitz <dunitzm@gmail.com>

backend/common/compute/diffexp_generic.py

@@ -25,7 +25,8 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
     :param maskB: observation selection mask for set 2
     :param top_n: number of variables to return stats for
     :param diffexp_lfc_cutoff: minimum
-    :return:  for top N genes, [ varindex, logfoldchange, pval, pval_adj ]
+    absolute value returning [ varindex, logfoldchange, pval, pval_adj ] for top N genes
+    :return:  for top N genes, {"positive": for top N genes, [ varindex, logfoldchange, pval, pval_adj ], "negative": for top N genes, [ varindex, logfoldchange, pval, pval_adj ]}
     """
 
     dataA = adaptor.get_X_array(maskA, None)
@@ -66,24 +67,27 @@ def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp
     # logfoldchanges: log2(meanA / meanB)
     logfoldchanges = np.log2(np.abs((meanA + 1e-9) / (meanB + 1e-9)))
 
+    stats_to_sort = tscores
     # find all with lfc > cutoff
     lfc_above_cutoff_idx = np.nonzero(np.abs(logfoldchanges) > diffexp_lfc_cutoff)[0]
-    stats_to_sort = np.abs(tscores)
 
     # derive sort order
-    if lfc_above_cutoff_idx.shape[0] > top_n:
+    if lfc_above_cutoff_idx.shape[0] > top_n*2:
         # partition top N
-        rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], -top_n)[-top_n:]
+        rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], (top_n, -top_n))
-        t_partition = lfc_above_cutoff_idx[rel_t_partition]
+        rel_t_partition_top_n = np.concatenate((rel_t_partition[-top_n:], rel_t_partition[:top_n]))
+        t_partition = lfc_above_cutoff_idx[rel_t_partition_top_n]
         # sort the top N partition
         rel_sort_order = np.argsort(stats_to_sort[t_partition])[::-1]
         sort_order = t_partition[rel_sort_order]
     else:
         # partition and sort top N, ignoring lfc cutoff
-        partition = np.argpartition(stats_to_sort, -top_n)[-top_n:]
+        partition = np.argpartition(stats_to_sort, (top_n, -top_n))
-        rel_sort_order = np.argsort(stats_to_sort[partition])[::-1]
+        partition_top_n = np.concatenate((partition[-top_n:], partition[:top_n]))
+
+        rel_sort_order = np.argsort(stats_to_sort[partition_top_n])[::-1]
         indices = np.indices(stats_to_sort.shape)[0]
-        sort_order = indices[partition][rel_sort_order]
+        sort_order = indices[partition_top_n][rel_sort_order]
 
     # top n slice based upon sort order
     logfoldchanges_top_n = logfoldchanges[sort_order]
@@ -91,7 +95,11 @@ def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp
     pvals_adj_top_n = pvals_adj[sort_order]
 
     # varIndex, logfoldchange, pval, pval_adj
-    result = [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n)]
+    result = {"positive": [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in
+                           range(top_n)],
+              "negative": [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in
+                           range(-1, -1 - top_n, -1)], }
+
     return result
 
 

backend/czi_hosted/common/rest.py

@@ -260,7 +260,6 @@ def diffexp_obs_post(request, data_adaptor):
     try:
         # TODO: implement varfilter mode
         mode = DiffExpMode(args["mode"])
-
         if mode == DiffExpMode.VAR_FILTER or "varFilter" in args:
             return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, "varFilter not enabled")
 

backend/czi_hosted/compute/diffexp_cxg.py

@@ -4,7 +4,7 @@ import numpy as np
 from numba import jit
 
 from backend.czi_hosted.data_cxg.cxg_util import pack_selector_from_indices
-from backend.czi_hosted.compute.diffexp_generic import diffexp_ttest_from_mean_var, mean_var_n
+from backend.common.compute.diffexp_generic import diffexp_ttest_from_mean_var, mean_var_n
 from backend.common.errors import ComputeError
 
 """
@@ -115,14 +115,14 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
             meanB += X_col_shift
 
     r = diffexp_ttest_from_mean_var(
-        meanA.astype(dtype),
+        meanA=meanA.astype(dtype),
-        varA.astype(dtype),
+        varA=varA.astype(dtype),
-        nA,
+        nA=nA,
-        meanB.astype(dtype),
+        meanB=meanB.astype(dtype),
-        varB.astype(dtype),
+        varB=varB.astype(dtype),
-        nB,
+        nB=nB,
-        top_n,
+        top_n=top_n,
-        diffexp_lfc_cutoff,
+        diffexp_lfc_cutoff=diffexp_lfc_cutoff
     )
 
     return r

backend/czi_hosted/data_anndata/anndata_adaptor.py

@@ -8,7 +8,7 @@ from pandas.core.dtypes.dtypes import CategoricalDtype
 from scipy import sparse
 from server_timing import Timing as ServerTiming
 
-import backend.czi_hosted.compute.diffexp_generic as diffexp_generic
+import backend.common.compute.diffexp_generic as diffexp_generic
 from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
 from backend.common.constants import Axis, MAX_LAYOUTS
 from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata

backend/czi_hosted/data_common/data_adaptor.py

@@ -163,7 +163,7 @@ class DataAdaptor(metaclass=ABCMeta):
         mask = np.zeros((count,), dtype=np.bool)
         for i in filter:
             if type(i) == list:
-                mask[i[0] : i[1]] = True
+                mask[i[0]: i[1]] = True
             else:
                 mask[i] = True
         return mask
@@ -321,11 +321,12 @@ class DataAdaptor(metaclass=ABCMeta):
             top_n = self.dataset_config.diffexp__top_n
 
         if self.server_config.exceeds_limit(
-            "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
+                "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
         ):
             raise ExceedsLimitError("Diffexp request exceeds max cell count limit")
 
-        result = self.compute_diffexp_ttest(obs_mask_A, obs_mask_B, top_n, self.dataset_config.diffexp__lfc_cutoff)
+        result = self.compute_diffexp_ttest(
+            maskA=obs_mask_A, maskB=obs_mask_B, top_n=top_n, lfc_cutoff=self.dataset_config.diffexp__lfc_cutoff)
 
         try:
             return jsonify_numpy(result)

backend/czi_hosted/data_cxg/cxg_adaptor.py

@@ -207,7 +207,8 @@ class CxgAdaptor(DataAdaptor):
             top_n = self.dataset_config.diffexp__top_n
         if lfc_cutoff is None:
             lfc_cutoff = self.dataset_config.diffexp__lfc_cutoff
-        return diffexp_cxg.diffexp_ttest(self, maskA, maskB, top_n, lfc_cutoff)
+        return diffexp_cxg.diffexp_ttest(
+            adaptor=self, maskA=maskA, maskB=maskB, top_n=top_n, diffexp_lfc_cutoff=lfc_cutoff)
 
     def get_colors(self):
         if self.cxg_version == "0.0":

backend/server/compute/diffexp_generic.py

@@ -1,134 +0,0 @@
-import numpy as np
-from scipy import sparse, stats
-
-
-def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
-    """
-    Return differential expression statistics for top N variables.
-
-    Algorithm:
-    - compute log fold change (log2(meanA/meanB))
-    - compute Welch's t-test statistic and pvalue (w/ Bonferroni correction)
-    - return top N abs(logfoldchange) where lfc > diffexp_lfc_cutoff
-
-    If there are not N which meet criteria, augment by removing the logfoldchange
-    threshold requirement.
-
-    Notes on alogrithm:
-    - Welch's ttest provides basic statistics test.
-      https://en.wikipedia.org/wiki/Welch%27s_t-test
-    - p-values adjusted with Bonferroni correction.
-      https://en.wikipedia.org/wiki/Bonferroni_correction
-
-    :param adaptor: DataAdaptor instance
-    :param maskA: observation selection mask for set 1
-    :param maskB: observation selection mask for set 2
-    :param top_n: number of variables to return stats for
-    :param diffexp_lfc_cutoff: minimum
-    :return:  for top N genes, [ varindex, logfoldchange, pval, pval_adj ]
-    """
-
-    dataA = adaptor.get_X_array(maskA, None)
-    dataB = adaptor.get_X_array(maskB, None)
-
-    # mean, variance, N - calculate for both selections
-    meanA, vA, nA = mean_var_n(dataA)
-    meanB, vB, nB = mean_var_n(dataB)
-    res = diffexp_ttest_from_mean_var(meanA, vA, nA, meanB, vB, nB, top_n, diffexp_lfc_cutoff)
-
-    return res
-
-
-def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp_lfc_cutoff):
-    n_var = meanA.shape[0]
-    top_n = min(top_n, n_var)
-
-    # variance / N
-    vnA = varA / min(nA, nB)  # overestimate variance, would normally be nA
-    vnB = varB / min(nA, nB)  # overestimate variance, would normally be nB
-    sum_vn = vnA + vnB
-
-    # degrees of freedom for Welch's t-test
-    with np.errstate(divide="ignore", invalid="ignore"):
-        dof = sum_vn ** 2 / (vnA ** 2 / (nA - 1) + vnB ** 2 / (nB - 1))
-    dof[np.isnan(dof)] = 1
-
-    # Welch's t-test score calculation
-    with np.errstate(divide="ignore", invalid="ignore"):
-        tscores = (meanA - meanB) / np.sqrt(sum_vn)
-    tscores[np.isnan(tscores)] = 0
-
-    # p-value
-    pvals = stats.t.sf(np.abs(tscores), dof) * 2
-    pvals_adj = pvals * n_var
-    pvals_adj[pvals_adj > 1] = 1  # cap adjusted p-value at 1
-
-    # logfoldchanges: log2(meanA / meanB)
-    logfoldchanges = np.log2(np.abs((meanA + 1e-9) / (meanB + 1e-9)))
-
-    # find all with lfc > cutoff
-    lfc_above_cutoff_idx = np.nonzero(np.abs(logfoldchanges) > diffexp_lfc_cutoff)[0]
-    stats_to_sort = np.abs(tscores)
-
-    # derive sort order
-    if lfc_above_cutoff_idx.shape[0] > top_n:
-        # partition top N
-        rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], -top_n)[-top_n:]
-        t_partition = lfc_above_cutoff_idx[rel_t_partition]
-        # sort the top N partition
-        rel_sort_order = np.argsort(stats_to_sort[t_partition])[::-1]
-        sort_order = t_partition[rel_sort_order]
-    else:
-        # partition and sort top N, ignoring lfc cutoff
-        partition = np.argpartition(stats_to_sort, -top_n)[-top_n:]
-        rel_sort_order = np.argsort(stats_to_sort[partition])[::-1]
-        indices = np.indices(stats_to_sort.shape)[0]
-        sort_order = indices[partition][rel_sort_order]
-
-    # top n slice based upon sort order
-    logfoldchanges_top_n = logfoldchanges[sort_order]
-    pvals_top_n = pvals[sort_order]
-    pvals_adj_top_n = pvals_adj[sort_order]
-
-    # varIndex, logfoldchange, pval, pval_adj
-    result = [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n)]
-    return result
-
-
-# Convenience function which handles sparse data
-def mean_var_n(X):
-    """
-    Two-pass variance calculation.  Numerically (more) stable
-    than naive methods (and same method used by numpy.var())
-    https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Two-pass
-    """
-    # fp_err_occurred is a flag indicating that a floating point error
-    # occured somewhere in our compute.  Used to trigger non-finite
-    # number handling.
-    fp_err_occurred = False
-
-    def fp_err_set(err, flag):
-        nonlocal fp_err_occurred
-        fp_err_occurred = True
-
-    with np.errstate(divide="call", invalid="call", call=fp_err_set):
-        n = X.shape[0]
-        if sparse.issparse(X):
-            mean = X.mean(axis=0).A1
-            dfm = X - mean
-            sumsq = np.sum(np.multiply(dfm, dfm), axis=0).A1
-            v = sumsq / (n - 1)
-        else:
-            mean = X.mean(axis=0)
-            dfm = X - mean
-            sumsq = np.sum(np.multiply(dfm, dfm), axis=0)
-            v = sumsq / (n - 1)
-
-    if fp_err_occurred:
-        mean[np.isfinite(mean) == False] = 0  # noqa: E712
-        v[np.isfinite(v) == False] = 0  # noqa: E712
-    else:
-        mean[np.isnan(mean)] = 0
-        v[np.isnan(v)] = 0
-
-    return mean, v, n

backend/server/data_anndata/anndata_adaptor.py

@@ -8,7 +8,7 @@ from pandas.core.dtypes.dtypes import CategoricalDtype
 from scipy import sparse
 from server_timing import Timing as ServerTiming
 
-import backend.server.compute.diffexp_generic as diffexp_generic
+import backend.common.compute.diffexp_generic as diffexp_generic
 from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
 from backend.common.constants import Axis, MAX_LAYOUTS
 from backend.server.common.corpora import corpora_get_props_from_anndata

backend/server/data_common/data_adaptor.py

@@ -68,7 +68,7 @@ class DataAdaptor(metaclass=ABCMeta):
 
     @abstractmethod
     def compute_embedding(self, method, filter):
-        """compute a new embedding on the specified obs subset, and return the embedding schema. """
+        """compute a new embedding on the specified obs subset, and return the embedding schema."""
         pass
 
     @abstractmethod
@@ -324,7 +324,12 @@ class DataAdaptor(metaclass=ABCMeta):
         ):
             raise ExceedsLimitError("Diffexp request exceeds max cell count limit")
 
-        result = self.compute_diffexp_ttest(obs_mask_A, obs_mask_B, top_n, self.dataset_config.diffexp__lfc_cutoff)
+        result = self.compute_diffexp_ttest(
+            maskA=obs_mask_A,
+            maskB=obs_mask_B,
+            top_n=top_n,
+            lfc_cutoff=self.dataset_config.diffexp__lfc_cutoff,
+        )
 
         try:
             return jsonify_numpy(result)

backend/test/test_czi_hosted/performance/run_diffexp.py

@@ -5,7 +5,8 @@ import time
 import numpy as np
 
 from backend.czi_hosted.common.config.app_config import AppConfig
-from backend.czi_hosted.compute import diffexp_generic, diffexp_cxg
+from backend.czi_hosted.compute import diffexp_cxg
+from backend.common.compute import diffexp_generic
 from backend.czi_hosted.data_common.matrix_loader import MatrixDataLoader
 from backend.czi_hosted.data_cxg.cxg_adaptor import CxgAdaptor
 

backend/test/test_czi_hosted/unit/common/test_api.py

@@ -158,7 +158,8 @@ class EndPoints(object):
         self.assertEqual(result.status_code, HTTPStatus.OK)
         self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data = result.json()
-        self.assertEqual(len(result_data), 7)
+        self.assertEqual(len(result_data['positive']), 7)
+        self.assertEqual(len(result_data['negative']), 7)
 
     def test_diff_exp_indices(self):
         endpoint = "diffexp/obs"
@@ -173,7 +174,8 @@ class EndPoints(object):
         self.assertEqual(result.status_code, HTTPStatus.OK)
         self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data = result.json()
-        self.assertEqual(len(result_data), 10)
+        self.assertEqual(len(result_data['positive']), 10)
+        self.assertEqual(len(result_data['negative']), 10)
 
     def test_get_annotations_var_fbs(self):
         endpoint = "annotations/var"
@@ -382,6 +384,7 @@ class EndPoints(object):
         query_hash = hashlib.sha1(query.encode()).hexdigest()
         url = f"{self.URL_BASE}{endpoint}?key={query_hash}"
         result = self.session.post(url, headers=headers, data=query)
+
         self.assertEqual(result.status_code, HTTPStatus.OK)
         self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
         df = decode_fbs.decode_matrix_FBS(result.content)

backend/test/test_czi_hosted/unit/compute/test_diffexp_cxg.py

@@ -4,7 +4,8 @@ import unittest
 
 import numpy as np
 
-from backend.czi_hosted.compute import diffexp_generic, diffexp_cxg
+from backend.czi_hosted.compute import diffexp_cxg
+from backend.common.compute import diffexp_generic
 from backend.czi_hosted.compute.diffexp_cxg import diffexp_ttest
 from backend.czi_hosted.converters.h5ad_data_file import H5ADDataFile
 from backend.common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
@@ -40,21 +41,37 @@ class DiffExpTest(unittest.TestCase):
             self.assertTrue(np.isclose(result[2], expect[2], 1e-6, 1e-4))
             self.assertTrue(np.isclose(result[3], expect[3], 1e-6, 1e-4))
 
+
     def check_1_10_2_10(self, results):
         """Checks the results for a specific set of rows selections"""
-        expects = [
+
+        positive_expects = [
+            [1712, -0.5525154, 0.0051788902660723345, 1.0],
+            [1575, 1.0317602, 0.007830310753043345, 1.0],
+            [693, 0.4703904, 0.008715846769131548, 1.0],
+            [916, 0.9567287, 0.009080596532247588, 1.0],
+            [77, 0.02665649, 0.010070392939027756, 1.0],
+            [782, -1.0981874, 0.010161745218916036, 1.0],
+            [913, 0.5683986, 0.010782030711612685, 1.0],
+            [910, 0.83164597, 0.014596411069229197, 1.0],
+            [1727, 0.4127781, 0.015168372104237176, 1.0],
+            [1443, -0.8241895, 0.015337080567465522, 1.0]
+        ]
+        negative_expects = [
             [956, 0.016060986, 0.0008649321884808977, 1.0],
             [1124, 0.96602094, 0.0011717216548271284, 1.0],
             [1809, 1.1110606, 0.0019304405196777848, 1.0],
-            [1712, -0.5525154, 0.0051788902660723345, 1.0],
             [1754, 0.5201581, 0.005691734062127954, 1.0],
             [948, 1.6390722, 0.006622111055981219, 1.0],
             [1810, 0.78618884, 0.007055917428377063, 1.0],
             [779, 1.5241305, 0.007202934422407284, 1.0],
-            [1575, 1.0317602, 0.007830310753043345, 1.0],
             [576, 0.97873515, 0.008272092578813124, 1.0],
+            [538, 0.89114505, 0.01062259019889307, 1.0],
+            [436, 0.3119122, 0.01127515110543434, 1.0]
         ]
-        self.compare_diffexp_results(results, expects)
+
+        self.compare_diffexp_results(results['positive'], positive_expects)
+        self.compare_diffexp_results(results['negative'], negative_expects)
 
     def get_X_col(self, adaptor, cols):
         varmask = np.zeros(adaptor.get_shape()[1], dtype=bool)
@@ -80,6 +97,7 @@ class DiffExpTest(unittest.TestCase):
         self.check_1_10_2_10(results)
 
         # run it directly
+
         results = diffexp_ttest(adaptor, maskA, maskB, 10)
         self.check_1_10_2_10(results)
 
@@ -128,15 +146,22 @@ class DiffExpTest(unittest.TestCase):
             diffexp_results_sparse = diffexp_cxg.diffexp_ttest(adaptor_sparse, maskA, maskB, 10)
             diffexp_results_dense = diffexp_cxg.diffexp_ttest(adaptor_dense, maskA, maskB, 10)
 
-            self.compare_diffexp_results(diffexp_results_anndata, diffexp_results_sparse)
+            self.compare_diffexp_results(diffexp_results_anndata['positive'], diffexp_results_sparse['positive'])
-            self.compare_diffexp_results(diffexp_results_anndata, diffexp_results_dense)
+            self.compare_diffexp_results(diffexp_results_anndata['negative'], diffexp_results_sparse['negative'])
+
+            self.compare_diffexp_results(diffexp_results_anndata['positive'], diffexp_results_dense['positive'])
+            self.compare_diffexp_results(diffexp_results_anndata['negative'], diffexp_results_dense['negative'])
+
+            topcols_pos = np.array([x[0] for x in diffexp_results_anndata['positive']])
+            topcols_neg = np.array([x[0] for x in diffexp_results_anndata['negative']])
+            topcols = np.concatenate((topcols_pos, topcols_neg))
 
-            topcols = np.array([x[0] for x in diffexp_results_anndata])
             cols_anndata = self.get_X_col(adaptor_anndata, topcols)
             cols_sparse = self.get_X_col(adaptor_sparse, topcols)
             cols_dense = self.get_X_col(adaptor_dense, topcols)
+
             assert cols_anndata.shape[0] == adaptor_sparse.get_shape()[0]
-            assert cols_anndata.shape[1] == len(diffexp_results_anndata)
+            assert cols_anndata.shape[1] == len(diffexp_results_anndata['positive']) + len(diffexp_results_anndata['negative'])
 
             def convert(mat, cols):
                 return decode_matrix_fbs(encode_matrix_fbs(mat, col_idx=cols)).to_numpy()

backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor.py

@@ -152,9 +152,11 @@ class AdaptorTest(unittest.TestCase):
         f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
         f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
         result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
-        self.assertEqual(len(result), 10)
+        self.assertEqual(len(result['positive']), 10)
+        self.assertEqual(len(result['negative']), 10)
         result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
-        self.assertEqual(len(result), 20)
+        self.assertEqual(len(result['positive']), 20)
+        self.assertEqual(len(result['negative']), 20)
 
     def test_data_frame(self):
         f1 = {"var": {"index": [[0, 10]]}}

backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor_data_load.py

@@ -30,10 +30,15 @@ class DataLoadAdaptorTest(unittest.TestCase):
     def test_diffexp_topN(self):
         f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
         f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
+
         result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
-        self.assertEqual(len(result), 10)
+
+        self.assertEqual(len(result['positive']), 10)
+        self.assertEqual(len(result['negative']), 10)
+
         result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
-        self.assertEqual(len(result), 20)
+        self.assertEqual(len(result['positive']), 20)
+        self.assertEqual(len(result['negative']), 20)
 
 
 class DataLocatorAdaptorTest(unittest.TestCase):

backend/test/test_server/performance/run_diffexp.py

@@ -4,7 +4,7 @@ import random
 import time
 import numpy as np
 
-import backend.server.compute.diffexp_generic as diffexp_generic
+import backend.common.compute.diffexp_generic as diffexp_generic
 
 from backend.server.common.config.app_config import AppConfig
 from backend.server.data_common.matrix_loader import MatrixDataLoader

backend/test/test_server/unit/common/test_api.py

@@ -414,7 +414,8 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
         self.assertEqual(result.status_code, HTTPStatus.OK)
         self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data = result.json()
-        self.assertEqual(len(result_data), 7)
+        self.assertEqual(len(result_data['positive']), 7)
+        self.assertEqual(len(result_data['negative']), 7)
 
     def test_diff_exp_indices(self):
         endpoint = "diffexp/obs"
@@ -429,7 +430,8 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
         self.assertEqual(result.status_code, HTTPStatus.OK)
         self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data = result.json()
-        self.assertEqual(len(result_data), 10)
+        self.assertEqual(len(result_data['positive']), 10)
+        self.assertEqual(len(result_data['negative']), 10)
 
     def test_get_summaryvar(self):
         index_col_name = self.schema["schema"]["annotations"]["var"]["index"]

backend/test/test_server/unit/compute/test_diffexp_h5ad.py

@@ -2,13 +2,14 @@ import unittest
 
 import numpy as np
 
+from backend.common.compute import diffexp_generic
 from backend.server.data_common.matrix_loader import MatrixDataLoader
 from backend.test.test_server.unit import app_config
 from backend.test import PROJECT_ROOT
 
 
 class DiffExpTest(unittest.TestCase):
-    """Tests the diffexp returns the expected results for one test case, using different
+    """Tests the diffexp returns the expected results for one test case, using the h5ad
     adaptor types and different algorithms."""
 
     def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}):
@@ -35,19 +36,34 @@ class DiffExpTest(unittest.TestCase):
 
     def check_1_10_2_10(self, results):
         """Checks the results for a specific set of rows selections"""
-        expects = [
+
+        positive_expects = [
+            [1712, -0.5525154, 0.0051788902660723345, 1.0],
+            [1575, 1.0317602, 0.007830310753043345, 1.0],
+            [693, 0.4703904, 0.008715846769131548, 1.0],
+            [916, 0.9567287, 0.009080596532247588, 1.0],
+            [77, 0.02665649, 0.010070392939027756, 1.0],
+            [782, -1.0981874, 0.010161745218916036, 1.0],
+            [913, 0.5683986, 0.010782030711612685, 1.0],
+            [910, 0.83164597, 0.014596411069229197, 1.0],
+            [1727, 0.4127781, 0.015168372104237176, 1.0],
+            [1443, -0.8241895, 0.015337080567465522, 1.0]
+        ]
+        negative_expects = [
             [956, 0.016060986, 0.0008649321884808977, 1.0],
             [1124, 0.96602094, 0.0011717216548271284, 1.0],
             [1809, 1.1110606, 0.0019304405196777848, 1.0],
-            [1712, -0.5525154, 0.0051788902660723345, 1.0],
             [1754, 0.5201581, 0.005691734062127954, 1.0],
             [948, 1.6390722, 0.006622111055981219, 1.0],
             [1810, 0.78618884, 0.007055917428377063, 1.0],
             [779, 1.5241305, 0.007202934422407284, 1.0],
-            [1575, 1.0317602, 0.007830310753043345, 1.0],
             [576, 0.97873515, 0.008272092578813124, 1.0],
+            [538, 0.89114505, 0.01062259019889307, 1.0],
+            [436, 0.3119122, 0.01127515110543434, 1.0]
         ]
-        self.compare_diffexp_results(results, expects)
+
+        self.compare_diffexp_results(results["positive"], positive_expects)
+        self.compare_diffexp_results(results["negative"], negative_expects)
 
     def get_X_col(self, adaptor, cols):
         varmask = np.zeros(adaptor.get_shape()[1], dtype=bool)
@@ -61,3 +77,29 @@ class DiffExpTest(unittest.TestCase):
         maskB = self.get_mask(adaptor, 2, 10)
         results = adaptor.compute_diffexp_ttest(maskA, maskB, 10)
         self.check_1_10_2_10(results)
+
+
+def test_h5ad_default(self):
+    """Test a h5ad adaptor with its default diffexp algorithm (diffexp_cxg)"""
+    adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
+    maskA = self.get_mask(adaptor, 1, 10)
+    maskB = self.get_mask(adaptor, 2, 10)
+
+    # run it through the adaptor
+    results = adaptor.compute_diffexp_ttest(maskA, maskB, 10)
+    self.check_1_10_2_10(results)
+
+    # run it directly
+
+    results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB, 10)
+    self.check_1_10_2_10(results)
+
+
+def test_h5ad_generic(self):
+    """Test a h5ad adaptor with the generic adaptor"""
+    adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
+    maskA = self.get_mask(adaptor, 1, 10)
+    maskB = self.get_mask(adaptor, 2, 10)
+    # run it directly
+    results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB, 10)
+    self.check_1_10_2_10(results)

backend/test/test_server/unit/data_anndata/test_anndata_adaptor.py

@@ -153,9 +153,12 @@ class AdaptorTest(unittest.TestCase):
         f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
         f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
         result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
-        self.assertEqual(len(result), 10)
+        self.assertEqual(len(result['positive']), 10)
+        self.assertEqual(len(result['negative']), 10)
+
         result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
-        self.assertEqual(len(result), 20)
+        self.assertEqual(len(result['positive']), 20)
+        self.assertEqual(len(result['negative']), 20)
 
     def test_data_frame(self):
         f1 = {"var": {"index": [[0, 10]]}}

backend/test/test_server/unit/data_anndata/test_anndata_adaptor_data_load.py

@@ -31,9 +31,11 @@ class DataLoadAdaptorTest(unittest.TestCase):
         f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
         f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
         result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
-        self.assertEqual(len(result), 10)
+        self.assertEqual(len(result['positive']), 10)
+        self.assertEqual(len(result['negative']), 10)
         result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
-        self.assertEqual(len(result), 20)
+        self.assertEqual(len(result['positive']), 20)
+        self.assertEqual(len(result['negative']), 20)
 
 
 class DataLocatorAdaptorTest(unittest.TestCase):

client/src/actions/index.js

@@ -211,15 +211,18 @@ const requestDifferentialExpression = (set1, set2, num_genes = 50) => async (
 
     const response = await res.json();
     const varIndex = await annoMatrix.fetch("var", varIndexName);
-    const data = response.map((v) => [
+    const diffexpLists = { negative: [], positive: [] };
-      varIndex.at(v[0], varIndexName),
+    for (const polarity of Object.keys(diffexpLists)) {
-      ...v.slice(1),
+      diffexpLists[polarity] = response[polarity].map((v) => [
-    ]);
+        varIndex.at(v[0], varIndexName),
+        ...v.slice(1),
+      ]);
+    }
 
     /* then send the success case action through */
     return dispatch({
       type: "request differential expression success",
-      data,
+      data: diffexpLists,
     });
   } catch (error) {
     return dispatch({

client/src/components/brushableHistogram/histogram.js

@@ -26,7 +26,13 @@ const Histogram = ({
     /*
       Create the d3 histogram
       */
-    const { marginLeft, marginRight, marginBottom, marginTop } = margin;
+    //  This is just a constant that's flipped by parent's `mini` boolean
+    const {
+      LEFT: marginLeft,
+      RIGHT: marginRight,
+      BOTTOM: marginBottom,
+      TOP: marginTop,
+    } = margin;
     const { x, y, bins, binStart, binEnd, binWidth } = histogram;
     const svg = d3.select(svgRef.current);
     const binPadding = mini ? 0 : -1;

client/src/components/brushableHistogram/index.js

@@ -13,6 +13,23 @@ import HistogramFooter from "./footer";
 import StillLoading from "./loading";
 import ErrorLoading from "./error";
 
+const MARGIN = {
+  LEFT: 10, // Space for 0 tick label on X axis
+  RIGHT: 54, // space for Y axis & labels
+  BOTTOM: 25, // space for X axis & labels
+  TOP: 3,
+};
+const WIDTH = 340 - MARGIN.LEFT - MARGIN.RIGHT;
+const HEIGHT = 135 - MARGIN.TOP - MARGIN.BOTTOM;
+const MARGIN_MINI = {
+  LEFT: 0, // Space for 0 tick label on X axis
+  RIGHT: 0, // space for Y axis & labels
+  BOTTOM: 0, // space for X axis & labels
+  TOP: 0,
+};
+const WIDTH_MINI = 120 - MARGIN_MINI.LEFT - MARGIN_MINI.RIGHT;
+const HEIGHT_MINI = 15 - MARGIN_MINI.TOP - MARGIN_MINI.BOTTOM;
+
 @connect((state, ownProps) => {
   const { isObs, isUserDefined, isGeneSetSummary, field } = ownProps;
   const myName = makeContinuousDimensionName(
@@ -44,34 +61,6 @@ class HistogramBrush extends React.PureComponent {
     }
   });
 
-  constructor(props) {
-    super(props);
-
-    const marginLeft = 10; // Space for 0 tick label on X axis
-    const marginRight = 54; // space for Y axis & labels
-    const marginBottom = 25; // space for X axis & labels
-    const marginTop = 3;
-
-    this.state = {
-      margin: {
-        marginLeft,
-        marginRight,
-        marginBottom,
-        marginTop,
-      },
-      width: 340 - marginLeft - marginRight,
-      height: 135 - marginTop - marginBottom,
-      marginMini: {
-        marginLeft: 0, // Space for 0 tick label on X axis
-        marginRight: 0, // space for Y axis & labels
-        marginBottom: 0, // space for X axis & labels
-        marginTop: 0,
-      },
-      widthMini: 120,
-      heightMini: 15,
-    };
-  }
-
   onBrush = (selection, x, eventType) => {
     const type = `continuous metadata histogram ${eventType}`;
     return () => {
@@ -210,15 +199,8 @@ class HistogramBrush extends React.PureComponent {
   };
 
   fetchAsyncProps = async () => {
-    const { annoMatrix } = this.props;
+    const { annoMatrix, width } = this.props;
-    const {
+
-      margin,
-      width,
-      height,
-      marginMini,
-      widthMini,
-      heightMini,
-    } = this.state;
     const { isClipped } = annoMatrix;
 
     const query = this.createQuery();
@@ -246,12 +228,17 @@ class HistogramBrush extends React.PureComponent {
         : globals.blue,
     ];
 
-    const histogram = this.calcHistogramCache(column, margin, width, height);
+    const histogram = this.calcHistogramCache(
+      column,
+      MARGIN,
+      width || WIDTH,
+      HEIGHT
+    );
     const miniHistogram = this.calcHistogramCache(
       column,
-      marginMini,
+      MARGIN_MINI,
-      widthMini,
+      width || WIDTH_MINI,
-      heightMini
+      HEIGHT_MINI
     );
 
     const isSingleValue = summary.min === summary.max;
@@ -275,7 +262,7 @@ class HistogramBrush extends React.PureComponent {
   };
 
   // eslint-disable-next-line class-methods-use-this -- instance method allows for memoization per annotation
-  calcHistogramCache(col, margin, width, height) {
+  calcHistogramCache(col, newMargin, newWidth, newHeight) {
     /*
      recalculate expensive stuff, notably bins, summaries, etc.
     */
@@ -283,7 +270,10 @@ class HistogramBrush extends React.PureComponent {
     const summary = col.summarize(); /* this is memoized, so it's free the second time you call it */
     const { min: domainMin, max: domainMax } = summary;
     const numBins = 40;
-    const { marginTop, marginLeft } = margin; /* changes with mini */
+    const {
+      TOP: topMargin,
+      LEFT: leftMargin,
+    } = newMargin; /* changes with mini */
 
     histogramCache.domain = [
       domainMin,
@@ -293,7 +283,7 @@ class HistogramBrush extends React.PureComponent {
     histogramCache.x = d3
       .scaleLinear()
       .domain([domainMin, domainMax])
-      .range([marginLeft, marginLeft + width]);
+      .range([leftMargin, leftMargin + newWidth]);
 
     histogramCache.bins = histogramContinuous(col, numBins, [
       domainMin,
@@ -310,7 +300,7 @@ class HistogramBrush extends React.PureComponent {
     histogramCache.y = d3
       .scaleLinear()
       .domain([0, yMax])
-      .range([marginTop + height, marginTop]);
+      .range([topMargin + newHeight, topMargin]);
 
     return histogramCache;
   }
@@ -367,14 +357,12 @@ class HistogramBrush extends React.PureComponent {
       mini,
       setGenes,
     } = this.props;
-    const {
+
-      margin,
+    let { width } = this.props;
-      width,
+    if (!width) {
-      height,
+      width = mini ? WIDTH_MINI : WIDTH;
-      marginMini,
+    }
-      widthMini,
+
-      heightMini,
-    } = this.state;
     const fieldForId = field.replace(/\s/g, "_");
     const showScatterPlot = isUserDefined;
 
@@ -432,11 +420,11 @@ class HistogramBrush extends React.PureComponent {
                   histogram={
                     mini ? asyncProps.miniHistogram : asyncProps.histogram
                   }
-                  width={mini ? widthMini : width}
+                  width={width}
-                  height={mini ? heightMini : height}
+                  height={mini ? HEIGHT_MINI : HEIGHT}
                   onBrush={this.onBrush}
                   onBrushEnd={this.onBrushEnd}
-                  margin={mini ? marginMini : margin}
+                  margin={mini ? MARGIN_MINI : MARGIN}
                   isColorBy={isColorAccessor}
                   selectionRange={continuousSelectionRange}
                   mini={mini}

client/src/components/geneExpression/gene.js

@@ -1,13 +1,14 @@
 import React from "react";
 import { connect } from "react-redux";
 
-import { AnchorButton, Icon } from "@blueprintjs/core";
+import { Button, Icon } from "@blueprintjs/core";
 import Truncate from "../util/truncate";
 import HistogramBrush from "../brushableHistogram";
 
-import * as globals from "../../globals";
 import actions from "../../actions";
 
+const MINI_HISTOGRAM_WIDTH = 110;
+
 @connect((state, ownProps) => {
   const { gene } = ownProps;
 
@@ -65,7 +66,7 @@ class Gene extends React.Component {
       isScatterplotYYaccessor,
     } = this.props;
     const { geneIsExpanded } = this.state;
-    const genesetNameLengthVisible = 310; /* this magic number determines how much of a long geneset name we see */
+    const geneSymbolWidth = 60 + (geneIsExpanded ? MINI_HISTOGRAM_WIDTH : 0);
 
     return (
       <div>
@@ -108,7 +109,8 @@ class Gene extends React.Component {
               >
                 <span
                   style={{
-                    width: globals.leftSidebarWidth - genesetNameLengthVisible,
+                    width: geneSymbolWidth,
+                    display: "inline-block",
                   }}
                   data-testid={`${gene}:gene-label`}
                 >
@@ -117,11 +119,16 @@ class Gene extends React.Component {
               </Truncate>
             </div>
             {!geneIsExpanded ? (
-              <HistogramBrush isUserDefined field={gene} mini />
+              <HistogramBrush
+                isUserDefined
+                field={gene}
+                mini
+                width={MINI_HISTOGRAM_WIDTH}
+              />
             ) : null}
           </div>
           <div style={{ flexShrink: 0, marginLeft: 2 }}>
-            <AnchorButton
+            <Button
               minimal
               small
               data-testid={`delete-from-geneset-${gene}`}
@@ -130,8 +137,7 @@ class Gene extends React.Component {
               style={{ fontWeight: 700, marginRight: 2 }}
               icon={<Icon icon="trash" iconSize={10} />}
             />
-            )
+            <Button
-            <AnchorButton
               minimal
               small
               data-testid={`plot-x-${gene}`}
@@ -141,8 +147,8 @@ class Gene extends React.Component {
               style={{ fontWeight: 700, marginRight: 2 }}
             >
               x
-            </AnchorButton>
+            </Button>
-            <AnchorButton
+            <Button
               minimal
               small
               data-testid={`plot-y-${gene}`}
@@ -152,8 +158,8 @@ class Gene extends React.Component {
               style={{ fontWeight: 700, marginRight: 2 }}
             >
               y
-            </AnchorButton>
+            </Button>
-            <AnchorButton
+            <Button
               minimal
               small
               data-testclass="maximize"
@@ -164,7 +170,7 @@ class Gene extends React.Component {
               icon={<Icon icon="maximize" iconSize={10} />}
               style={{ marginRight: 2 }}
             />
-            <AnchorButton
+            <Button
               minimal
               small
               data-testclass="colorby"

client/src/reducers/genesets.js

@@ -96,12 +96,17 @@ const GeneSets = (
       if (state.genesets.has(genesetName))
         throw new Error("geneset: create -- name already defined.");
 
-      const genesets = new Map(state.genesets); // clone
+      const genesets = new Map([
-      genesets.set(genesetName, {
+        [
-        genesetName,
+          genesetName,
-        genesetDescription,
+          {
-        genes: new Map(),
+            genesetName,
-      });
+            genesetDescription,
+            genes: new Map(),
+          },
+        ],
+        ...state.genesets,
+      ]); // clone and add new geneset to beginning
 
       return {
         ...state,
@@ -357,23 +362,34 @@ const GeneSets = (
     case "request differential expression success": {
       const { data } = action;
 
-      const genes = new Map(
+      const dateString = new Date().toLocaleString();
-        data.map((diffExpGene) => [
+
-          diffExpGene[0],
+      const genesetNames = {
+        positive: `Pop1 high (${dateString})`,
+        negative: `Pop2 high (${dateString})`,
+      };
+
+      const diffExpGeneSets = [];
+      for (const polarity of Object.keys(genesetNames)) {
+        const genes = new Map(
+          data[polarity].map((diffExpGene) => [
+            diffExpGene[0],
+            {
+              geneSymbol: diffExpGene[0],
+            },
+          ])
+        );
+        diffExpGeneSets.push([
+          genesetNames[polarity],
           {
-            geneSymbol: diffExpGene[0],
+            genesetName: genesetNames[polarity],
+            genesetDescription: "",
+            genes,
           },
-        ])
+        ]);
-      );
+      }
 
-      const genesetName = `DiffExp Set (${new Date().toLocaleString()})`;
+      const genesets = new Map([...diffExpGeneSets, ...state.genesets]); // clone
-
-      const genesets = new Map(state.genesets); // clone
-      genesets.set(genesetName, {
-        genesetName,
-        genesetDescription: "",
-        genes,
-      });
 
       return {
         ...state,