From 28b526b3fc75a04c03dea29093e5de29b5bc6e01 Mon Sep 17 00:00:00 2001 From: Severiano Badajoz Date: Tue, 8 Jun 2021 14:02:19 -0700 Subject: [PATCH] feat: diffexp returns two genesets (#2230) * feat: return two lists for diffexp (#2221) * sp * split out derive sort order, tests passing * sp * return diff exp results in two lists * update * copy implementation over to desktop * add tests for two lists * small fixes to complete backend implementation * accept new diffexp response * map diff exp response to genesets * delete ) * name diffexp genesets with population names * take constants out of state and allow width prop to override * shorten mini-histo properly truncate and resize depending on expansion * prepend new genesets * rename data within diffexp action * backend * move diffexp ttest to common code module, update tests * update for unit tests * reference actual var Co-authored-by: Madison Dunitz Co-authored-by: Madison Dunitz --- backend/common/compute/__init__.py | 0 .../compute/diffexp_generic.py | 26 ++-- backend/czi_hosted/common/rest.py | 1 - backend/czi_hosted/compute/diffexp_cxg.py | 18 +-- .../data_anndata/anndata_adaptor.py | 2 +- .../czi_hosted/data_common/data_adaptor.py | 7 +- backend/czi_hosted/data_cxg/cxg_adaptor.py | 3 +- backend/server/compute/diffexp_generic.py | 134 ------------------ .../server/data_anndata/anndata_adaptor.py | 2 +- backend/server/data_common/data_adaptor.py | 9 +- .../performance/run_diffexp.py | 3 +- .../test_czi_hosted/unit/common/test_api.py | 7 +- .../unit/compute/test_diffexp_cxg.py | 43 ++++-- .../unit/data_anndata/test_anndata_adaptor.py | 6 +- .../test_anndata_adaptor_data_load.py | 9 +- .../test_server/performance/run_diffexp.py | 2 +- .../test/test_server/unit/common/test_api.py | 6 +- ...st_diffexp_cxg.py => test_diffexp_h5ad.py} | 52 ++++++- .../unit/data_anndata/test_anndata_adaptor.py | 7 +- .../test_anndata_adaptor_data_load.py | 6 +- client/src/actions/index.js | 13 +- .../brushableHistogram/histogram.js | 8 +- .../components/brushableHistogram/index.js | 100 ++++++------- client/src/components/geneExpression/gene.js | 32 +++-- client/src/reducers/genesets.js | 56 +++++--- 25 files changed, 268 insertions(+), 284 deletions(-) create mode 100644 backend/common/compute/__init__.py rename backend/{czi_hosted => common}/compute/diffexp_generic.py (79%) delete mode 100644 backend/server/compute/diffexp_generic.py rename backend/test/test_server/unit/compute/{test_diffexp_cxg.py => test_diffexp_h5ad.py} (59%) diff --git a/backend/common/compute/__init__.py b/backend/common/compute/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/czi_hosted/compute/diffexp_generic.py b/backend/common/compute/diffexp_generic.py similarity index 79% rename from backend/czi_hosted/compute/diffexp_generic.py rename to backend/common/compute/diffexp_generic.py index 71fe9a59..be6f7ded 100644 --- a/backend/czi_hosted/compute/diffexp_generic.py +++ b/backend/common/compute/diffexp_generic.py @@ -25,7 +25,8 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01): :param maskB: observation selection mask for set 2 :param top_n: number of variables to return stats for :param diffexp_lfc_cutoff: minimum - :return: for top N genes, [ varindex, logfoldchange, pval, pval_adj ] + absolute value returning [ varindex, logfoldchange, pval, pval_adj ] for top N genes + :return: for top N genes, {"positive": for top N genes, [ varindex, logfoldchange, pval, pval_adj ], "negative": for top N genes, [ varindex, logfoldchange, pval, pval_adj ]} """ dataA = adaptor.get_X_array(maskA, None) @@ -66,24 +67,27 @@ def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp # logfoldchanges: log2(meanA / meanB) logfoldchanges = np.log2(np.abs((meanA + 1e-9) / (meanB + 1e-9))) + stats_to_sort = tscores # find all with lfc > cutoff lfc_above_cutoff_idx = np.nonzero(np.abs(logfoldchanges) > diffexp_lfc_cutoff)[0] - stats_to_sort = np.abs(tscores) # derive sort order - if lfc_above_cutoff_idx.shape[0] > top_n: + if lfc_above_cutoff_idx.shape[0] > top_n*2: # partition top N - rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], -top_n)[-top_n:] - t_partition = lfc_above_cutoff_idx[rel_t_partition] + rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], (top_n, -top_n)) + rel_t_partition_top_n = np.concatenate((rel_t_partition[-top_n:], rel_t_partition[:top_n])) + t_partition = lfc_above_cutoff_idx[rel_t_partition_top_n] # sort the top N partition rel_sort_order = np.argsort(stats_to_sort[t_partition])[::-1] sort_order = t_partition[rel_sort_order] else: # partition and sort top N, ignoring lfc cutoff - partition = np.argpartition(stats_to_sort, -top_n)[-top_n:] - rel_sort_order = np.argsort(stats_to_sort[partition])[::-1] + partition = np.argpartition(stats_to_sort, (top_n, -top_n)) + partition_top_n = np.concatenate((partition[-top_n:], partition[:top_n])) + + rel_sort_order = np.argsort(stats_to_sort[partition_top_n])[::-1] indices = np.indices(stats_to_sort.shape)[0] - sort_order = indices[partition][rel_sort_order] + sort_order = indices[partition_top_n][rel_sort_order] # top n slice based upon sort order logfoldchanges_top_n = logfoldchanges[sort_order] @@ -91,7 +95,11 @@ def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp pvals_adj_top_n = pvals_adj[sort_order] # varIndex, logfoldchange, pval, pval_adj - result = [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n)] + result = {"positive": [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in + range(top_n)], + "negative": [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in + range(-1, -1 - top_n, -1)], } + return result diff --git a/backend/czi_hosted/common/rest.py b/backend/czi_hosted/common/rest.py index f9980744..a138f8c5 100644 --- a/backend/czi_hosted/common/rest.py +++ b/backend/czi_hosted/common/rest.py @@ -260,7 +260,6 @@ def diffexp_obs_post(request, data_adaptor): try: # TODO: implement varfilter mode mode = DiffExpMode(args["mode"]) - if mode == DiffExpMode.VAR_FILTER or "varFilter" in args: return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, "varFilter not enabled") diff --git a/backend/czi_hosted/compute/diffexp_cxg.py b/backend/czi_hosted/compute/diffexp_cxg.py index 15c385e7..d37d866c 100644 --- a/backend/czi_hosted/compute/diffexp_cxg.py +++ b/backend/czi_hosted/compute/diffexp_cxg.py @@ -4,7 +4,7 @@ import numpy as np from numba import jit from backend.czi_hosted.data_cxg.cxg_util import pack_selector_from_indices -from backend.czi_hosted.compute.diffexp_generic import diffexp_ttest_from_mean_var, mean_var_n +from backend.common.compute.diffexp_generic import diffexp_ttest_from_mean_var, mean_var_n from backend.common.errors import ComputeError """ @@ -115,14 +115,14 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01): meanB += X_col_shift r = diffexp_ttest_from_mean_var( - meanA.astype(dtype), - varA.astype(dtype), - nA, - meanB.astype(dtype), - varB.astype(dtype), - nB, - top_n, - diffexp_lfc_cutoff, + meanA=meanA.astype(dtype), + varA=varA.astype(dtype), + nA=nA, + meanB=meanB.astype(dtype), + varB=varB.astype(dtype), + nB=nB, + top_n=top_n, + diffexp_lfc_cutoff=diffexp_lfc_cutoff ) return r diff --git a/backend/czi_hosted/data_anndata/anndata_adaptor.py b/backend/czi_hosted/data_anndata/anndata_adaptor.py index 76311fb9..177e9c15 100644 --- a/backend/czi_hosted/data_anndata/anndata_adaptor.py +++ b/backend/czi_hosted/data_anndata/anndata_adaptor.py @@ -8,7 +8,7 @@ from pandas.core.dtypes.dtypes import CategoricalDtype from scipy import sparse from server_timing import Timing as ServerTiming -import backend.czi_hosted.compute.diffexp_generic as diffexp_generic +import backend.common.compute.diffexp_generic as diffexp_generic from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors from backend.common.constants import Axis, MAX_LAYOUTS from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata diff --git a/backend/czi_hosted/data_common/data_adaptor.py b/backend/czi_hosted/data_common/data_adaptor.py index 9b80289c..d652523e 100644 --- a/backend/czi_hosted/data_common/data_adaptor.py +++ b/backend/czi_hosted/data_common/data_adaptor.py @@ -163,7 +163,7 @@ class DataAdaptor(metaclass=ABCMeta): mask = np.zeros((count,), dtype=np.bool) for i in filter: if type(i) == list: - mask[i[0] : i[1]] = True + mask[i[0]: i[1]] = True else: mask[i] = True return mask @@ -321,11 +321,12 @@ class DataAdaptor(metaclass=ABCMeta): top_n = self.dataset_config.diffexp__top_n if self.server_config.exceeds_limit( - "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B) + "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B) ): raise ExceedsLimitError("Diffexp request exceeds max cell count limit") - result = self.compute_diffexp_ttest(obs_mask_A, obs_mask_B, top_n, self.dataset_config.diffexp__lfc_cutoff) + result = self.compute_diffexp_ttest( + maskA=obs_mask_A, maskB=obs_mask_B, top_n=top_n, lfc_cutoff=self.dataset_config.diffexp__lfc_cutoff) try: return jsonify_numpy(result) diff --git a/backend/czi_hosted/data_cxg/cxg_adaptor.py b/backend/czi_hosted/data_cxg/cxg_adaptor.py index f09f06a6..c773aa48 100644 --- a/backend/czi_hosted/data_cxg/cxg_adaptor.py +++ b/backend/czi_hosted/data_cxg/cxg_adaptor.py @@ -207,7 +207,8 @@ class CxgAdaptor(DataAdaptor): top_n = self.dataset_config.diffexp__top_n if lfc_cutoff is None: lfc_cutoff = self.dataset_config.diffexp__lfc_cutoff - return diffexp_cxg.diffexp_ttest(self, maskA, maskB, top_n, lfc_cutoff) + return diffexp_cxg.diffexp_ttest( + adaptor=self, maskA=maskA, maskB=maskB, top_n=top_n, diffexp_lfc_cutoff=lfc_cutoff) def get_colors(self): if self.cxg_version == "0.0": diff --git a/backend/server/compute/diffexp_generic.py b/backend/server/compute/diffexp_generic.py deleted file mode 100644 index 71fe9a59..00000000 --- a/backend/server/compute/diffexp_generic.py +++ /dev/null @@ -1,134 +0,0 @@ -import numpy as np -from scipy import sparse, stats - - -def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01): - """ - Return differential expression statistics for top N variables. - - Algorithm: - - compute log fold change (log2(meanA/meanB)) - - compute Welch's t-test statistic and pvalue (w/ Bonferroni correction) - - return top N abs(logfoldchange) where lfc > diffexp_lfc_cutoff - - If there are not N which meet criteria, augment by removing the logfoldchange - threshold requirement. - - Notes on alogrithm: - - Welch's ttest provides basic statistics test. - https://en.wikipedia.org/wiki/Welch%27s_t-test - - p-values adjusted with Bonferroni correction. - https://en.wikipedia.org/wiki/Bonferroni_correction - - :param adaptor: DataAdaptor instance - :param maskA: observation selection mask for set 1 - :param maskB: observation selection mask for set 2 - :param top_n: number of variables to return stats for - :param diffexp_lfc_cutoff: minimum - :return: for top N genes, [ varindex, logfoldchange, pval, pval_adj ] - """ - - dataA = adaptor.get_X_array(maskA, None) - dataB = adaptor.get_X_array(maskB, None) - - # mean, variance, N - calculate for both selections - meanA, vA, nA = mean_var_n(dataA) - meanB, vB, nB = mean_var_n(dataB) - res = diffexp_ttest_from_mean_var(meanA, vA, nA, meanB, vB, nB, top_n, diffexp_lfc_cutoff) - - return res - - -def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp_lfc_cutoff): - n_var = meanA.shape[0] - top_n = min(top_n, n_var) - - # variance / N - vnA = varA / min(nA, nB) # overestimate variance, would normally be nA - vnB = varB / min(nA, nB) # overestimate variance, would normally be nB - sum_vn = vnA + vnB - - # degrees of freedom for Welch's t-test - with np.errstate(divide="ignore", invalid="ignore"): - dof = sum_vn ** 2 / (vnA ** 2 / (nA - 1) + vnB ** 2 / (nB - 1)) - dof[np.isnan(dof)] = 1 - - # Welch's t-test score calculation - with np.errstate(divide="ignore", invalid="ignore"): - tscores = (meanA - meanB) / np.sqrt(sum_vn) - tscores[np.isnan(tscores)] = 0 - - # p-value - pvals = stats.t.sf(np.abs(tscores), dof) * 2 - pvals_adj = pvals * n_var - pvals_adj[pvals_adj > 1] = 1 # cap adjusted p-value at 1 - - # logfoldchanges: log2(meanA / meanB) - logfoldchanges = np.log2(np.abs((meanA + 1e-9) / (meanB + 1e-9))) - - # find all with lfc > cutoff - lfc_above_cutoff_idx = np.nonzero(np.abs(logfoldchanges) > diffexp_lfc_cutoff)[0] - stats_to_sort = np.abs(tscores) - - # derive sort order - if lfc_above_cutoff_idx.shape[0] > top_n: - # partition top N - rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], -top_n)[-top_n:] - t_partition = lfc_above_cutoff_idx[rel_t_partition] - # sort the top N partition - rel_sort_order = np.argsort(stats_to_sort[t_partition])[::-1] - sort_order = t_partition[rel_sort_order] - else: - # partition and sort top N, ignoring lfc cutoff - partition = np.argpartition(stats_to_sort, -top_n)[-top_n:] - rel_sort_order = np.argsort(stats_to_sort[partition])[::-1] - indices = np.indices(stats_to_sort.shape)[0] - sort_order = indices[partition][rel_sort_order] - - # top n slice based upon sort order - logfoldchanges_top_n = logfoldchanges[sort_order] - pvals_top_n = pvals[sort_order] - pvals_adj_top_n = pvals_adj[sort_order] - - # varIndex, logfoldchange, pval, pval_adj - result = [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n)] - return result - - -# Convenience function which handles sparse data -def mean_var_n(X): - """ - Two-pass variance calculation. Numerically (more) stable - than naive methods (and same method used by numpy.var()) - https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Two-pass - """ - # fp_err_occurred is a flag indicating that a floating point error - # occured somewhere in our compute. Used to trigger non-finite - # number handling. - fp_err_occurred = False - - def fp_err_set(err, flag): - nonlocal fp_err_occurred - fp_err_occurred = True - - with np.errstate(divide="call", invalid="call", call=fp_err_set): - n = X.shape[0] - if sparse.issparse(X): - mean = X.mean(axis=0).A1 - dfm = X - mean - sumsq = np.sum(np.multiply(dfm, dfm), axis=0).A1 - v = sumsq / (n - 1) - else: - mean = X.mean(axis=0) - dfm = X - mean - sumsq = np.sum(np.multiply(dfm, dfm), axis=0) - v = sumsq / (n - 1) - - if fp_err_occurred: - mean[np.isfinite(mean) == False] = 0 # noqa: E712 - v[np.isfinite(v) == False] = 0 # noqa: E712 - else: - mean[np.isnan(mean)] = 0 - v[np.isnan(v)] = 0 - - return mean, v, n diff --git a/backend/server/data_anndata/anndata_adaptor.py b/backend/server/data_anndata/anndata_adaptor.py index ebf47c29..81439892 100644 --- a/backend/server/data_anndata/anndata_adaptor.py +++ b/backend/server/data_anndata/anndata_adaptor.py @@ -8,7 +8,7 @@ from pandas.core.dtypes.dtypes import CategoricalDtype from scipy import sparse from server_timing import Timing as ServerTiming -import backend.server.compute.diffexp_generic as diffexp_generic +import backend.common.compute.diffexp_generic as diffexp_generic from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors from backend.common.constants import Axis, MAX_LAYOUTS from backend.server.common.corpora import corpora_get_props_from_anndata diff --git a/backend/server/data_common/data_adaptor.py b/backend/server/data_common/data_adaptor.py index 310ead61..720e5166 100644 --- a/backend/server/data_common/data_adaptor.py +++ b/backend/server/data_common/data_adaptor.py @@ -68,7 +68,7 @@ class DataAdaptor(metaclass=ABCMeta): @abstractmethod def compute_embedding(self, method, filter): - """compute a new embedding on the specified obs subset, and return the embedding schema. """ + """compute a new embedding on the specified obs subset, and return the embedding schema.""" pass @abstractmethod @@ -324,7 +324,12 @@ class DataAdaptor(metaclass=ABCMeta): ): raise ExceedsLimitError("Diffexp request exceeds max cell count limit") - result = self.compute_diffexp_ttest(obs_mask_A, obs_mask_B, top_n, self.dataset_config.diffexp__lfc_cutoff) + result = self.compute_diffexp_ttest( + maskA=obs_mask_A, + maskB=obs_mask_B, + top_n=top_n, + lfc_cutoff=self.dataset_config.diffexp__lfc_cutoff, + ) try: return jsonify_numpy(result) diff --git a/backend/test/test_czi_hosted/performance/run_diffexp.py b/backend/test/test_czi_hosted/performance/run_diffexp.py index 27a9e40f..cb6fa847 100644 --- a/backend/test/test_czi_hosted/performance/run_diffexp.py +++ b/backend/test/test_czi_hosted/performance/run_diffexp.py @@ -5,7 +5,8 @@ import time import numpy as np from backend.czi_hosted.common.config.app_config import AppConfig -from backend.czi_hosted.compute import diffexp_generic, diffexp_cxg +from backend.czi_hosted.compute import diffexp_cxg +from backend.common.compute import diffexp_generic from backend.czi_hosted.data_common.matrix_loader import MatrixDataLoader from backend.czi_hosted.data_cxg.cxg_adaptor import CxgAdaptor diff --git a/backend/test/test_czi_hosted/unit/common/test_api.py b/backend/test/test_czi_hosted/unit/common/test_api.py index 00bce5b2..ee729750 100644 --- a/backend/test/test_czi_hosted/unit/common/test_api.py +++ b/backend/test/test_czi_hosted/unit/common/test_api.py @@ -158,7 +158,8 @@ class EndPoints(object): self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.headers["Content-Type"], "application/json") result_data = result.json() - self.assertEqual(len(result_data), 7) + self.assertEqual(len(result_data['positive']), 7) + self.assertEqual(len(result_data['negative']), 7) def test_diff_exp_indices(self): endpoint = "diffexp/obs" @@ -173,7 +174,8 @@ class EndPoints(object): self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.headers["Content-Type"], "application/json") result_data = result.json() - self.assertEqual(len(result_data), 10) + self.assertEqual(len(result_data['positive']), 10) + self.assertEqual(len(result_data['negative']), 10) def test_get_annotations_var_fbs(self): endpoint = "annotations/var" @@ -382,6 +384,7 @@ class EndPoints(object): query_hash = hashlib.sha1(query.encode()).hexdigest() url = f"{self.URL_BASE}{endpoint}?key={query_hash}" result = self.session.post(url, headers=headers, data=query) + self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.headers["Content-Type"], "application/octet-stream") df = decode_fbs.decode_matrix_FBS(result.content) diff --git a/backend/test/test_czi_hosted/unit/compute/test_diffexp_cxg.py b/backend/test/test_czi_hosted/unit/compute/test_diffexp_cxg.py index 20500b42..8e0e29e2 100644 --- a/backend/test/test_czi_hosted/unit/compute/test_diffexp_cxg.py +++ b/backend/test/test_czi_hosted/unit/compute/test_diffexp_cxg.py @@ -4,7 +4,8 @@ import unittest import numpy as np -from backend.czi_hosted.compute import diffexp_generic, diffexp_cxg +from backend.czi_hosted.compute import diffexp_cxg +from backend.common.compute import diffexp_generic from backend.czi_hosted.compute.diffexp_cxg import diffexp_ttest from backend.czi_hosted.converters.h5ad_data_file import H5ADDataFile from backend.common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs @@ -40,21 +41,37 @@ class DiffExpTest(unittest.TestCase): self.assertTrue(np.isclose(result[2], expect[2], 1e-6, 1e-4)) self.assertTrue(np.isclose(result[3], expect[3], 1e-6, 1e-4)) + def check_1_10_2_10(self, results): """Checks the results for a specific set of rows selections""" - expects = [ + + positive_expects = [ + [1712, -0.5525154, 0.0051788902660723345, 1.0], + [1575, 1.0317602, 0.007830310753043345, 1.0], + [693, 0.4703904, 0.008715846769131548, 1.0], + [916, 0.9567287, 0.009080596532247588, 1.0], + [77, 0.02665649, 0.010070392939027756, 1.0], + [782, -1.0981874, 0.010161745218916036, 1.0], + [913, 0.5683986, 0.010782030711612685, 1.0], + [910, 0.83164597, 0.014596411069229197, 1.0], + [1727, 0.4127781, 0.015168372104237176, 1.0], + [1443, -0.8241895, 0.015337080567465522, 1.0] + ] + negative_expects = [ [956, 0.016060986, 0.0008649321884808977, 1.0], [1124, 0.96602094, 0.0011717216548271284, 1.0], [1809, 1.1110606, 0.0019304405196777848, 1.0], - [1712, -0.5525154, 0.0051788902660723345, 1.0], [1754, 0.5201581, 0.005691734062127954, 1.0], [948, 1.6390722, 0.006622111055981219, 1.0], [1810, 0.78618884, 0.007055917428377063, 1.0], [779, 1.5241305, 0.007202934422407284, 1.0], - [1575, 1.0317602, 0.007830310753043345, 1.0], [576, 0.97873515, 0.008272092578813124, 1.0], + [538, 0.89114505, 0.01062259019889307, 1.0], + [436, 0.3119122, 0.01127515110543434, 1.0] ] - self.compare_diffexp_results(results, expects) + + self.compare_diffexp_results(results['positive'], positive_expects) + self.compare_diffexp_results(results['negative'], negative_expects) def get_X_col(self, adaptor, cols): varmask = np.zeros(adaptor.get_shape()[1], dtype=bool) @@ -80,6 +97,7 @@ class DiffExpTest(unittest.TestCase): self.check_1_10_2_10(results) # run it directly + results = diffexp_ttest(adaptor, maskA, maskB, 10) self.check_1_10_2_10(results) @@ -128,15 +146,22 @@ class DiffExpTest(unittest.TestCase): diffexp_results_sparse = diffexp_cxg.diffexp_ttest(adaptor_sparse, maskA, maskB, 10) diffexp_results_dense = diffexp_cxg.diffexp_ttest(adaptor_dense, maskA, maskB, 10) - self.compare_diffexp_results(diffexp_results_anndata, diffexp_results_sparse) - self.compare_diffexp_results(diffexp_results_anndata, diffexp_results_dense) + self.compare_diffexp_results(diffexp_results_anndata['positive'], diffexp_results_sparse['positive']) + self.compare_diffexp_results(diffexp_results_anndata['negative'], diffexp_results_sparse['negative']) + + self.compare_diffexp_results(diffexp_results_anndata['positive'], diffexp_results_dense['positive']) + self.compare_diffexp_results(diffexp_results_anndata['negative'], diffexp_results_dense['negative']) + + topcols_pos = np.array([x[0] for x in diffexp_results_anndata['positive']]) + topcols_neg = np.array([x[0] for x in diffexp_results_anndata['negative']]) + topcols = np.concatenate((topcols_pos, topcols_neg)) - topcols = np.array([x[0] for x in diffexp_results_anndata]) cols_anndata = self.get_X_col(adaptor_anndata, topcols) cols_sparse = self.get_X_col(adaptor_sparse, topcols) cols_dense = self.get_X_col(adaptor_dense, topcols) + assert cols_anndata.shape[0] == adaptor_sparse.get_shape()[0] - assert cols_anndata.shape[1] == len(diffexp_results_anndata) + assert cols_anndata.shape[1] == len(diffexp_results_anndata['positive']) + len(diffexp_results_anndata['negative']) def convert(mat, cols): return decode_matrix_fbs(encode_matrix_fbs(mat, col_idx=cols)).to_numpy() diff --git a/backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor.py b/backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor.py index 0089b8fb..16252a79 100644 --- a/backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor.py +++ b/backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor.py @@ -152,9 +152,11 @@ class AdaptorTest(unittest.TestCase): f1 = {"filter": {"obs": {"index": [[0, 500]]}}} f2 = {"filter": {"obs": {"index": [[500, 1000]]}}} result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"])) - self.assertEqual(len(result), 10) + self.assertEqual(len(result['positive']), 10) + self.assertEqual(len(result['negative']), 10) result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20)) - self.assertEqual(len(result), 20) + self.assertEqual(len(result['positive']), 20) + self.assertEqual(len(result['negative']), 20) def test_data_frame(self): f1 = {"var": {"index": [[0, 10]]}} diff --git a/backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor_data_load.py b/backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor_data_load.py index 297e455a..9a258198 100644 --- a/backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor_data_load.py +++ b/backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor_data_load.py @@ -30,10 +30,15 @@ class DataLoadAdaptorTest(unittest.TestCase): def test_diffexp_topN(self): f1 = {"filter": {"obs": {"index": [[0, 500]]}}} f2 = {"filter": {"obs": {"index": [[500, 1000]]}}} + result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"])) - self.assertEqual(len(result), 10) + + self.assertEqual(len(result['positive']), 10) + self.assertEqual(len(result['negative']), 10) + result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20)) - self.assertEqual(len(result), 20) + self.assertEqual(len(result['positive']), 20) + self.assertEqual(len(result['negative']), 20) class DataLocatorAdaptorTest(unittest.TestCase): diff --git a/backend/test/test_server/performance/run_diffexp.py b/backend/test/test_server/performance/run_diffexp.py index ee2413aa..9475fac4 100644 --- a/backend/test/test_server/performance/run_diffexp.py +++ b/backend/test/test_server/performance/run_diffexp.py @@ -4,7 +4,7 @@ import random import time import numpy as np -import backend.server.compute.diffexp_generic as diffexp_generic +import backend.common.compute.diffexp_generic as diffexp_generic from backend.server.common.config.app_config import AppConfig from backend.server.data_common.matrix_loader import MatrixDataLoader diff --git a/backend/test/test_server/unit/common/test_api.py b/backend/test/test_server/unit/common/test_api.py index 19ac7873..9c8984e4 100644 --- a/backend/test/test_server/unit/common/test_api.py +++ b/backend/test/test_server/unit/common/test_api.py @@ -414,7 +414,8 @@ class EndPointsAnndata(unittest.TestCase, EndPoints): self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.headers["Content-Type"], "application/json") result_data = result.json() - self.assertEqual(len(result_data), 7) + self.assertEqual(len(result_data['positive']), 7) + self.assertEqual(len(result_data['negative']), 7) def test_diff_exp_indices(self): endpoint = "diffexp/obs" @@ -429,7 +430,8 @@ class EndPointsAnndata(unittest.TestCase, EndPoints): self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.headers["Content-Type"], "application/json") result_data = result.json() - self.assertEqual(len(result_data), 10) + self.assertEqual(len(result_data['positive']), 10) + self.assertEqual(len(result_data['negative']), 10) def test_get_summaryvar(self): index_col_name = self.schema["schema"]["annotations"]["var"]["index"] diff --git a/backend/test/test_server/unit/compute/test_diffexp_cxg.py b/backend/test/test_server/unit/compute/test_diffexp_h5ad.py similarity index 59% rename from backend/test/test_server/unit/compute/test_diffexp_cxg.py rename to backend/test/test_server/unit/compute/test_diffexp_h5ad.py index 0febada5..3f98a023 100644 --- a/backend/test/test_server/unit/compute/test_diffexp_cxg.py +++ b/backend/test/test_server/unit/compute/test_diffexp_h5ad.py @@ -2,13 +2,14 @@ import unittest import numpy as np +from backend.common.compute import diffexp_generic from backend.server.data_common.matrix_loader import MatrixDataLoader from backend.test.test_server.unit import app_config from backend.test import PROJECT_ROOT class DiffExpTest(unittest.TestCase): - """Tests the diffexp returns the expected results for one test case, using different + """Tests the diffexp returns the expected results for one test case, using the h5ad adaptor types and different algorithms.""" def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}): @@ -35,19 +36,34 @@ class DiffExpTest(unittest.TestCase): def check_1_10_2_10(self, results): """Checks the results for a specific set of rows selections""" - expects = [ + + positive_expects = [ + [1712, -0.5525154, 0.0051788902660723345, 1.0], + [1575, 1.0317602, 0.007830310753043345, 1.0], + [693, 0.4703904, 0.008715846769131548, 1.0], + [916, 0.9567287, 0.009080596532247588, 1.0], + [77, 0.02665649, 0.010070392939027756, 1.0], + [782, -1.0981874, 0.010161745218916036, 1.0], + [913, 0.5683986, 0.010782030711612685, 1.0], + [910, 0.83164597, 0.014596411069229197, 1.0], + [1727, 0.4127781, 0.015168372104237176, 1.0], + [1443, -0.8241895, 0.015337080567465522, 1.0] + ] + negative_expects = [ [956, 0.016060986, 0.0008649321884808977, 1.0], [1124, 0.96602094, 0.0011717216548271284, 1.0], [1809, 1.1110606, 0.0019304405196777848, 1.0], - [1712, -0.5525154, 0.0051788902660723345, 1.0], [1754, 0.5201581, 0.005691734062127954, 1.0], [948, 1.6390722, 0.006622111055981219, 1.0], [1810, 0.78618884, 0.007055917428377063, 1.0], [779, 1.5241305, 0.007202934422407284, 1.0], - [1575, 1.0317602, 0.007830310753043345, 1.0], [576, 0.97873515, 0.008272092578813124, 1.0], + [538, 0.89114505, 0.01062259019889307, 1.0], + [436, 0.3119122, 0.01127515110543434, 1.0] ] - self.compare_diffexp_results(results, expects) + + self.compare_diffexp_results(results["positive"], positive_expects) + self.compare_diffexp_results(results["negative"], negative_expects) def get_X_col(self, adaptor, cols): varmask = np.zeros(adaptor.get_shape()[1], dtype=bool) @@ -61,3 +77,29 @@ class DiffExpTest(unittest.TestCase): maskB = self.get_mask(adaptor, 2, 10) results = adaptor.compute_diffexp_ttest(maskA, maskB, 10) self.check_1_10_2_10(results) + + +def test_h5ad_default(self): + """Test a h5ad adaptor with its default diffexp algorithm (diffexp_cxg)""" + adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad") + maskA = self.get_mask(adaptor, 1, 10) + maskB = self.get_mask(adaptor, 2, 10) + + # run it through the adaptor + results = adaptor.compute_diffexp_ttest(maskA, maskB, 10) + self.check_1_10_2_10(results) + + # run it directly + + results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB, 10) + self.check_1_10_2_10(results) + + +def test_h5ad_generic(self): + """Test a h5ad adaptor with the generic adaptor""" + adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad") + maskA = self.get_mask(adaptor, 1, 10) + maskB = self.get_mask(adaptor, 2, 10) + # run it directly + results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB, 10) + self.check_1_10_2_10(results) diff --git a/backend/test/test_server/unit/data_anndata/test_anndata_adaptor.py b/backend/test/test_server/unit/data_anndata/test_anndata_adaptor.py index c3e6f665..f17e27f4 100644 --- a/backend/test/test_server/unit/data_anndata/test_anndata_adaptor.py +++ b/backend/test/test_server/unit/data_anndata/test_anndata_adaptor.py @@ -153,9 +153,12 @@ class AdaptorTest(unittest.TestCase): f1 = {"filter": {"obs": {"index": [[0, 500]]}}} f2 = {"filter": {"obs": {"index": [[500, 1000]]}}} result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"])) - self.assertEqual(len(result), 10) + self.assertEqual(len(result['positive']), 10) + self.assertEqual(len(result['negative']), 10) + result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20)) - self.assertEqual(len(result), 20) + self.assertEqual(len(result['positive']), 20) + self.assertEqual(len(result['negative']), 20) def test_data_frame(self): f1 = {"var": {"index": [[0, 10]]}} diff --git a/backend/test/test_server/unit/data_anndata/test_anndata_adaptor_data_load.py b/backend/test/test_server/unit/data_anndata/test_anndata_adaptor_data_load.py index c480d8d5..5e52a5d2 100644 --- a/backend/test/test_server/unit/data_anndata/test_anndata_adaptor_data_load.py +++ b/backend/test/test_server/unit/data_anndata/test_anndata_adaptor_data_load.py @@ -31,9 +31,11 @@ class DataLoadAdaptorTest(unittest.TestCase): f1 = {"filter": {"obs": {"index": [[0, 500]]}}} f2 = {"filter": {"obs": {"index": [[500, 1000]]}}} result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"])) - self.assertEqual(len(result), 10) + self.assertEqual(len(result['positive']), 10) + self.assertEqual(len(result['negative']), 10) result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20)) - self.assertEqual(len(result), 20) + self.assertEqual(len(result['positive']), 20) + self.assertEqual(len(result['negative']), 20) class DataLocatorAdaptorTest(unittest.TestCase): diff --git a/client/src/actions/index.js b/client/src/actions/index.js index 13c075e6..94f8c58b 100644 --- a/client/src/actions/index.js +++ b/client/src/actions/index.js @@ -211,15 +211,18 @@ const requestDifferentialExpression = (set1, set2, num_genes = 50) => async ( const response = await res.json(); const varIndex = await annoMatrix.fetch("var", varIndexName); - const data = response.map((v) => [ - varIndex.at(v[0], varIndexName), - ...v.slice(1), - ]); + const diffexpLists = { negative: [], positive: [] }; + for (const polarity of Object.keys(diffexpLists)) { + diffexpLists[polarity] = response[polarity].map((v) => [ + varIndex.at(v[0], varIndexName), + ...v.slice(1), + ]); + } /* then send the success case action through */ return dispatch({ type: "request differential expression success", - data, + data: diffexpLists, }); } catch (error) { return dispatch({ diff --git a/client/src/components/brushableHistogram/histogram.js b/client/src/components/brushableHistogram/histogram.js index e49eb6e4..a9ae1190 100644 --- a/client/src/components/brushableHistogram/histogram.js +++ b/client/src/components/brushableHistogram/histogram.js @@ -26,7 +26,13 @@ const Histogram = ({ /* Create the d3 histogram */ - const { marginLeft, marginRight, marginBottom, marginTop } = margin; + // This is just a constant that's flipped by parent's `mini` boolean + const { + LEFT: marginLeft, + RIGHT: marginRight, + BOTTOM: marginBottom, + TOP: marginTop, + } = margin; const { x, y, bins, binStart, binEnd, binWidth } = histogram; const svg = d3.select(svgRef.current); const binPadding = mini ? 0 : -1; diff --git a/client/src/components/brushableHistogram/index.js b/client/src/components/brushableHistogram/index.js index a9394c49..89773fa8 100644 --- a/client/src/components/brushableHistogram/index.js +++ b/client/src/components/brushableHistogram/index.js @@ -13,6 +13,23 @@ import HistogramFooter from "./footer"; import StillLoading from "./loading"; import ErrorLoading from "./error"; +const MARGIN = { + LEFT: 10, // Space for 0 tick label on X axis + RIGHT: 54, // space for Y axis & labels + BOTTOM: 25, // space for X axis & labels + TOP: 3, +}; +const WIDTH = 340 - MARGIN.LEFT - MARGIN.RIGHT; +const HEIGHT = 135 - MARGIN.TOP - MARGIN.BOTTOM; +const MARGIN_MINI = { + LEFT: 0, // Space for 0 tick label on X axis + RIGHT: 0, // space for Y axis & labels + BOTTOM: 0, // space for X axis & labels + TOP: 0, +}; +const WIDTH_MINI = 120 - MARGIN_MINI.LEFT - MARGIN_MINI.RIGHT; +const HEIGHT_MINI = 15 - MARGIN_MINI.TOP - MARGIN_MINI.BOTTOM; + @connect((state, ownProps) => { const { isObs, isUserDefined, isGeneSetSummary, field } = ownProps; const myName = makeContinuousDimensionName( @@ -44,34 +61,6 @@ class HistogramBrush extends React.PureComponent { } }); - constructor(props) { - super(props); - - const marginLeft = 10; // Space for 0 tick label on X axis - const marginRight = 54; // space for Y axis & labels - const marginBottom = 25; // space for X axis & labels - const marginTop = 3; - - this.state = { - margin: { - marginLeft, - marginRight, - marginBottom, - marginTop, - }, - width: 340 - marginLeft - marginRight, - height: 135 - marginTop - marginBottom, - marginMini: { - marginLeft: 0, // Space for 0 tick label on X axis - marginRight: 0, // space for Y axis & labels - marginBottom: 0, // space for X axis & labels - marginTop: 0, - }, - widthMini: 120, - heightMini: 15, - }; - } - onBrush = (selection, x, eventType) => { const type = `continuous metadata histogram ${eventType}`; return () => { @@ -210,15 +199,8 @@ class HistogramBrush extends React.PureComponent { }; fetchAsyncProps = async () => { - const { annoMatrix } = this.props; - const { - margin, - width, - height, - marginMini, - widthMini, - heightMini, - } = this.state; + const { annoMatrix, width } = this.props; + const { isClipped } = annoMatrix; const query = this.createQuery(); @@ -246,12 +228,17 @@ class HistogramBrush extends React.PureComponent { : globals.blue, ]; - const histogram = this.calcHistogramCache(column, margin, width, height); + const histogram = this.calcHistogramCache( + column, + MARGIN, + width || WIDTH, + HEIGHT + ); const miniHistogram = this.calcHistogramCache( column, - marginMini, - widthMini, - heightMini + MARGIN_MINI, + width || WIDTH_MINI, + HEIGHT_MINI ); const isSingleValue = summary.min === summary.max; @@ -275,7 +262,7 @@ class HistogramBrush extends React.PureComponent { }; // eslint-disable-next-line class-methods-use-this -- instance method allows for memoization per annotation - calcHistogramCache(col, margin, width, height) { + calcHistogramCache(col, newMargin, newWidth, newHeight) { /* recalculate expensive stuff, notably bins, summaries, etc. */ @@ -283,7 +270,10 @@ class HistogramBrush extends React.PureComponent { const summary = col.summarize(); /* this is memoized, so it's free the second time you call it */ const { min: domainMin, max: domainMax } = summary; const numBins = 40; - const { marginTop, marginLeft } = margin; /* changes with mini */ + const { + TOP: topMargin, + LEFT: leftMargin, + } = newMargin; /* changes with mini */ histogramCache.domain = [ domainMin, @@ -293,7 +283,7 @@ class HistogramBrush extends React.PureComponent { histogramCache.x = d3 .scaleLinear() .domain([domainMin, domainMax]) - .range([marginLeft, marginLeft + width]); + .range([leftMargin, leftMargin + newWidth]); histogramCache.bins = histogramContinuous(col, numBins, [ domainMin, @@ -310,7 +300,7 @@ class HistogramBrush extends React.PureComponent { histogramCache.y = d3 .scaleLinear() .domain([0, yMax]) - .range([marginTop + height, marginTop]); + .range([topMargin + newHeight, topMargin]); return histogramCache; } @@ -367,14 +357,12 @@ class HistogramBrush extends React.PureComponent { mini, setGenes, } = this.props; - const { - margin, - width, - height, - marginMini, - widthMini, - heightMini, - } = this.state; + + let { width } = this.props; + if (!width) { + width = mini ? WIDTH_MINI : WIDTH; + } + const fieldForId = field.replace(/\s/g, "_"); const showScatterPlot = isUserDefined; @@ -432,11 +420,11 @@ class HistogramBrush extends React.PureComponent { histogram={ mini ? asyncProps.miniHistogram : asyncProps.histogram } - width={mini ? widthMini : width} - height={mini ? heightMini : height} + width={width} + height={mini ? HEIGHT_MINI : HEIGHT} onBrush={this.onBrush} onBrushEnd={this.onBrushEnd} - margin={mini ? marginMini : margin} + margin={mini ? MARGIN_MINI : MARGIN} isColorBy={isColorAccessor} selectionRange={continuousSelectionRange} mini={mini} diff --git a/client/src/components/geneExpression/gene.js b/client/src/components/geneExpression/gene.js index 5dcab06a..5912e912 100644 --- a/client/src/components/geneExpression/gene.js +++ b/client/src/components/geneExpression/gene.js @@ -1,13 +1,14 @@ import React from "react"; import { connect } from "react-redux"; -import { AnchorButton, Icon } from "@blueprintjs/core"; +import { Button, Icon } from "@blueprintjs/core"; import Truncate from "../util/truncate"; import HistogramBrush from "../brushableHistogram"; -import * as globals from "../../globals"; import actions from "../../actions"; +const MINI_HISTOGRAM_WIDTH = 110; + @connect((state, ownProps) => { const { gene } = ownProps; @@ -65,7 +66,7 @@ class Gene extends React.Component { isScatterplotYYaccessor, } = this.props; const { geneIsExpanded } = this.state; - const genesetNameLengthVisible = 310; /* this magic number determines how much of a long geneset name we see */ + const geneSymbolWidth = 60 + (geneIsExpanded ? MINI_HISTOGRAM_WIDTH : 0); return (
@@ -108,7 +109,8 @@ class Gene extends React.Component { > @@ -117,11 +119,16 @@ class Gene extends React.Component {
{!geneIsExpanded ? ( - + ) : null}
- } /> - ) - x - - +