fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-15 12:47:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: diffexp returns two genesets (#2230)

Browse Source 
* feat: return two lists for diffexp (#2221)

* sp

* split out derive sort order, tests passing

* sp

* return diff exp results in two lists

* update

* copy implementation over to desktop

* add tests for two lists

* small fixes to complete backend implementation

* accept new diffexp response

* map diff exp response to genesets

* delete )

* name diffexp genesets with population names

* take constants out of state and allow width prop to override

* shorten mini-histo properly truncate and resize depending on expansion

* prepend new genesets

* rename data within diffexp action

* backend

* move diffexp ttest to common code module, update tests

* update for unit tests

* reference actual var

Co-authored-by: Madison Dunitz <madison.dunitz@chanzuckerberg.com>
Co-authored-by: Madison Dunitz <dunitzm@gmail.com>
This commit is contained in:
Severiano Badajoz
2021-06-08 14:02:19 -07:00
committed by GitHub
parent 7ed53c0f5b
commit 28b526b3fc
25 changed files with 268 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
backend/common/compute/__init__.py Normal file
View File

26
backend/czi_hosted/compute/diffexp_generic.py → backend/common/compute/diffexp_generic.py
View File

@@ -25,7 +25,8 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
:param maskB: observation selection mask for set 2
:param top_n: number of variables to return stats for
:param diffexp_lfc_cutoff: minimum
:return: for top N genes, [ varindex, logfoldchange, pval, pval_adj ]
absolute value returning [ varindex, logfoldchange, pval, pval_adj ] for top N genes
:return: for top N genes, {"positive": for top N genes, [ varindex, logfoldchange, pval, pval_adj ], "negative": for top N genes, [ varindex, logfoldchange, pval, pval_adj ]}
"""
dataA = adaptor.get_X_array(maskA, None)
@@ -66,24 +67,27 @@ def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp
# logfoldchanges: log2(meanA / meanB)
logfoldchanges = np.log2(np.abs((meanA + 1e-9) / (meanB + 1e-9)))
stats_to_sort = tscores
# find all with lfc > cutoff
lfc_above_cutoff_idx = np.nonzero(np.abs(logfoldchanges) > diffexp_lfc_cutoff)[0]
stats_to_sort = np.abs(tscores)
# derive sort order
if lfc_above_cutoff_idx.shape[0] > top_n:
if lfc_above_cutoff_idx.shape[0] > top_n*2:
# partition top N
rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], -top_n)[-top_n:]
t_partition = lfc_above_cutoff_idx[rel_t_partition]
rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], (top_n, -top_n))
rel_t_partition_top_n = np.concatenate((rel_t_partition[-top_n:], rel_t_partition[:top_n]))
t_partition = lfc_above_cutoff_idx[rel_t_partition_top_n]
# sort the top N partition
rel_sort_order = np.argsort(stats_to_sort[t_partition])[::-1]
sort_order = t_partition[rel_sort_order]
else:
# partition and sort top N, ignoring lfc cutoff
partition = np.argpartition(stats_to_sort, -top_n)[-top_n:]
rel_sort_order = np.argsort(stats_to_sort[partition])[::-1]
partition = np.argpartition(stats_to_sort, (top_n, -top_n))
partition_top_n = np.concatenate((partition[-top_n:], partition[:top_n]))
rel_sort_order = np.argsort(stats_to_sort[partition_top_n])[::-1]
indices = np.indices(stats_to_sort.shape)[0]
sort_order = indices[partition][rel_sort_order]
sort_order = indices[partition_top_n][rel_sort_order]
# top n slice based upon sort order
logfoldchanges_top_n = logfoldchanges[sort_order]
@@ -91,7 +95,11 @@ def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp
pvals_adj_top_n = pvals_adj[sort_order]
# varIndex, logfoldchange, pval, pval_adj
result = [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n)]
result = {"positive": [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in
range(top_n)],
"negative": [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in
range(-1, -1 - top_n, -1)], }
return result

1
backend/czi_hosted/common/rest.py
View File

@@ -260,7 +260,6 @@ def diffexp_obs_post(request, data_adaptor):
try:
# TODO: implement varfilter mode
mode = DiffExpMode(args["mode"])
if mode == DiffExpMode.VAR_FILTER or "varFilter" in args:
return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, "varFilter not enabled")

18
backend/czi_hosted/compute/diffexp_cxg.py
View File

@@ -4,7 +4,7 @@ import numpy as np
from numba import jit
from backend.czi_hosted.data_cxg.cxg_util import pack_selector_from_indices
from backend.czi_hosted.compute.diffexp_generic import diffexp_ttest_from_mean_var, mean_var_n
from backend.common.compute.diffexp_generic import diffexp_ttest_from_mean_var, mean_var_n
from backend.common.errors import ComputeError
"""
@@ -115,14 +115,14 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
meanB += X_col_shift
r = diffexp_ttest_from_mean_var(
meanA.astype(dtype),
varA.astype(dtype),
nA,
meanB.astype(dtype),
varB.astype(dtype),
nB,
top_n,
diffexp_lfc_cutoff,
meanA=meanA.astype(dtype),
varA=varA.astype(dtype),
nA=nA,
meanB=meanB.astype(dtype),
varB=varB.astype(dtype),
nB=nB,
top_n=top_n,
diffexp_lfc_cutoff=diffexp_lfc_cutoff
)
return r

2
backend/czi_hosted/data_anndata/anndata_adaptor.py
View File

@@ -8,7 +8,7 @@ from pandas.core.dtypes.dtypes import CategoricalDtype
from scipy import sparse
from server_timing import Timing as ServerTiming
import backend.czi_hosted.compute.diffexp_generic as diffexp_generic
import backend.common.compute.diffexp_generic as diffexp_generic
from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
from backend.common.constants import Axis, MAX_LAYOUTS
from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata

7
backend/czi_hosted/data_common/data_adaptor.py
View File

@@ -163,7 +163,7 @@ class DataAdaptor(metaclass=ABCMeta):
mask = np.zeros((count,), dtype=np.bool)
for i in filter:
if type(i) == list:
mask[i[0] : i[1]] = True
mask[i[0]: i[1]] = True
else:
mask[i] = True
return mask
@@ -321,11 +321,12 @@ class DataAdaptor(metaclass=ABCMeta):
top_n = self.dataset_config.diffexp__top_n
if self.server_config.exceeds_limit(
"diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
"diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
):
raise ExceedsLimitError("Diffexp request exceeds max cell count limit")
result = self.compute_diffexp_ttest(obs_mask_A, obs_mask_B, top_n, self.dataset_config.diffexp__lfc_cutoff)
result = self.compute_diffexp_ttest(
maskA=obs_mask_A, maskB=obs_mask_B, top_n=top_n, lfc_cutoff=self.dataset_config.diffexp__lfc_cutoff)
try:
return jsonify_numpy(result)

3
backend/czi_hosted/data_cxg/cxg_adaptor.py
View File

@@ -207,7 +207,8 @@ class CxgAdaptor(DataAdaptor):
top_n = self.dataset_config.diffexp__top_n
if lfc_cutoff is None:
lfc_cutoff = self.dataset_config.diffexp__lfc_cutoff
return diffexp_cxg.diffexp_ttest(self, maskA, maskB, top_n, lfc_cutoff)
return diffexp_cxg.diffexp_ttest(
adaptor=self, maskA=maskA, maskB=maskB, top_n=top_n, diffexp_lfc_cutoff=lfc_cutoff)
def get_colors(self):
if self.cxg_version == "0.0":

134
backend/server/compute/diffexp_generic.py
View File

@@ -1,134 +0,0 @@
import numpy as np
from scipy import sparse, stats
def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
"""
Return differential expression statistics for top N variables.
Algorithm:
- compute log fold change (log2(meanA/meanB))
- compute Welch's t-test statistic and pvalue (w/ Bonferroni correction)
- return top N abs(logfoldchange) where lfc > diffexp_lfc_cutoff
If there are not N which meet criteria, augment by removing the logfoldchange
threshold requirement.
Notes on alogrithm:
- Welch's ttest provides basic statistics test.
https://en.wikipedia.org/wiki/Welch%27s_t-test
- p-values adjusted with Bonferroni correction.
https://en.wikipedia.org/wiki/Bonferroni_correction
:param adaptor: DataAdaptor instance
:param maskA: observation selection mask for set 1
:param maskB: observation selection mask for set 2
:param top_n: number of variables to return stats for
:param diffexp_lfc_cutoff: minimum
:return: for top N genes, [ varindex, logfoldchange, pval, pval_adj ]
"""
dataA = adaptor.get_X_array(maskA, None)
dataB = adaptor.get_X_array(maskB, None)
# mean, variance, N - calculate for both selections
meanA, vA, nA = mean_var_n(dataA)
meanB, vB, nB = mean_var_n(dataB)
res = diffexp_ttest_from_mean_var(meanA, vA, nA, meanB, vB, nB, top_n, diffexp_lfc_cutoff)
return res
def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp_lfc_cutoff):
n_var = meanA.shape[0]
top_n = min(top_n, n_var)
# variance / N
vnA = varA / min(nA, nB) # overestimate variance, would normally be nA
vnB = varB / min(nA, nB) # overestimate variance, would normally be nB
sum_vn = vnA + vnB
# degrees of freedom for Welch's t-test
with np.errstate(divide="ignore", invalid="ignore"):
dof = sum_vn ** 2 / (vnA ** 2 / (nA - 1) + vnB ** 2 / (nB - 1))
dof[np.isnan(dof)] = 1
# Welch's t-test score calculation
with np.errstate(divide="ignore", invalid="ignore"):
tscores = (meanA - meanB) / np.sqrt(sum_vn)
tscores[np.isnan(tscores)] = 0
# p-value
pvals = stats.t.sf(np.abs(tscores), dof) * 2
pvals_adj = pvals * n_var
pvals_adj[pvals_adj > 1] = 1 # cap adjusted p-value at 1
# logfoldchanges: log2(meanA / meanB)
logfoldchanges = np.log2(np.abs((meanA + 1e-9) / (meanB + 1e-9)))
# find all with lfc > cutoff
lfc_above_cutoff_idx = np.nonzero(np.abs(logfoldchanges) > diffexp_lfc_cutoff)[0]
stats_to_sort = np.abs(tscores)
# derive sort order
if lfc_above_cutoff_idx.shape[0] > top_n:
# partition top N
rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], -top_n)[-top_n:]
t_partition = lfc_above_cutoff_idx[rel_t_partition]
# sort the top N partition
rel_sort_order = np.argsort(stats_to_sort[t_partition])[::-1]
sort_order = t_partition[rel_sort_order]
else:
# partition and sort top N, ignoring lfc cutoff
partition = np.argpartition(stats_to_sort, -top_n)[-top_n:]
rel_sort_order = np.argsort(stats_to_sort[partition])[::-1]
indices = np.indices(stats_to_sort.shape)[0]
sort_order = indices[partition][rel_sort_order]
# top n slice based upon sort order
logfoldchanges_top_n = logfoldchanges[sort_order]
pvals_top_n = pvals[sort_order]
pvals_adj_top_n = pvals_adj[sort_order]
# varIndex, logfoldchange, pval, pval_adj
result = [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n)]
return result
# Convenience function which handles sparse data
def mean_var_n(X):
"""
Two-pass variance calculation. Numerically (more) stable
than naive methods (and same method used by numpy.var())
https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Two-pass
"""
# fp_err_occurred is a flag indicating that a floating point error
# occured somewhere in our compute. Used to trigger non-finite
# number handling.
fp_err_occurred = False
def fp_err_set(err, flag):
nonlocal fp_err_occurred
fp_err_occurred = True
with np.errstate(divide="call", invalid="call", call=fp_err_set):
n = X.shape[0]
if sparse.issparse(X):
mean = X.mean(axis=0).A1
dfm = X - mean
sumsq = np.sum(np.multiply(dfm, dfm), axis=0).A1
v = sumsq / (n - 1)
else:
mean = X.mean(axis=0)
dfm = X - mean
sumsq = np.sum(np.multiply(dfm, dfm), axis=0)
v = sumsq / (n - 1)
if fp_err_occurred:
mean[np.isfinite(mean) == False] = 0 # noqa: E712
v[np.isfinite(v) == False] = 0 # noqa: E712
else:
mean[np.isnan(mean)] = 0
v[np.isnan(v)] = 0
return mean, v, n

2
backend/server/data_anndata/anndata_adaptor.py
View File

@@ -8,7 +8,7 @@ from pandas.core.dtypes.dtypes import CategoricalDtype
from scipy import sparse
from server_timing import Timing as ServerTiming
import backend.server.compute.diffexp_generic as diffexp_generic
import backend.common.compute.diffexp_generic as diffexp_generic
from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
from backend.common.constants import Axis, MAX_LAYOUTS
from backend.server.common.corpora import corpora_get_props_from_anndata

9
backend/server/data_common/data_adaptor.py
View File

@@ -68,7 +68,7 @@ class DataAdaptor(metaclass=ABCMeta):
@abstractmethod
def compute_embedding(self, method, filter):
"""compute a new embedding on the specified obs subset, and return the embedding schema. """
"""compute a new embedding on the specified obs subset, and return the embedding schema."""
pass
@abstractmethod
@@ -324,7 +324,12 @@ class DataAdaptor(metaclass=ABCMeta):
):
raise ExceedsLimitError("Diffexp request exceeds max cell count limit")
result = self.compute_diffexp_ttest(obs_mask_A, obs_mask_B, top_n, self.dataset_config.diffexp__lfc_cutoff)
result = self.compute_diffexp_ttest(
maskA=obs_mask_A,
maskB=obs_mask_B,
top_n=top_n,
lfc_cutoff=self.dataset_config.diffexp__lfc_cutoff,
)
try:
return jsonify_numpy(result)

3
backend/test/test_czi_hosted/performance/run_diffexp.py
View File

@@ -5,7 +5,8 @@ import time
import numpy as np
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.czi_hosted.compute import diffexp_generic, diffexp_cxg
from backend.czi_hosted.compute import diffexp_cxg
from backend.common.compute import diffexp_generic
from backend.czi_hosted.data_common.matrix_loader import MatrixDataLoader
from backend.czi_hosted.data_cxg.cxg_adaptor import CxgAdaptor

7
backend/test/test_czi_hosted/unit/common/test_api.py
View File

@@ -158,7 +158,8 @@ class EndPoints(object):
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data), 7)
self.assertEqual(len(result_data['positive']), 7)
self.assertEqual(len(result_data['negative']), 7)
def test_diff_exp_indices(self):
endpoint = "diffexp/obs"
@@ -173,7 +174,8 @@ class EndPoints(object):
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data), 10)
self.assertEqual(len(result_data['positive']), 10)
self.assertEqual(len(result_data['negative']), 10)
def test_get_annotations_var_fbs(self):
endpoint = "annotations/var"
@@ -382,6 +384,7 @@ class EndPoints(object):
query_hash = hashlib.sha1(query.encode()).hexdigest()
url = f"{self.URL_BASE}{endpoint}?key={query_hash}"
result = self.session.post(url, headers=headers, data=query)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)

43
backend/test/test_czi_hosted/unit/compute/test_diffexp_cxg.py
View File

@@ -4,7 +4,8 @@ import unittest
import numpy as np
from backend.czi_hosted.compute import diffexp_generic, diffexp_cxg
from backend.czi_hosted.compute import diffexp_cxg
from backend.common.compute import diffexp_generic
from backend.czi_hosted.compute.diffexp_cxg import diffexp_ttest
from backend.czi_hosted.converters.h5ad_data_file import H5ADDataFile
from backend.common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
@@ -40,21 +41,37 @@ class DiffExpTest(unittest.TestCase):
self.assertTrue(np.isclose(result[2], expect[2], 1e-6, 1e-4))
self.assertTrue(np.isclose(result[3], expect[3], 1e-6, 1e-4))
def check_1_10_2_10(self, results):
"""Checks the results for a specific set of rows selections"""
expects = [
positive_expects = [
[1712, -0.5525154, 0.0051788902660723345, 1.0],
[1575, 1.0317602, 0.007830310753043345, 1.0],
[693, 0.4703904, 0.008715846769131548, 1.0],
[916, 0.9567287, 0.009080596532247588, 1.0],
[77, 0.02665649, 0.010070392939027756, 1.0],
[782, -1.0981874, 0.010161745218916036, 1.0],
[913, 0.5683986, 0.010782030711612685, 1.0],
[910, 0.83164597, 0.014596411069229197, 1.0],
[1727, 0.4127781, 0.015168372104237176, 1.0],
[1443, -0.8241895, 0.015337080567465522, 1.0]
]
negative_expects = [
[956, 0.016060986, 0.0008649321884808977, 1.0],
[1124, 0.96602094, 0.0011717216548271284, 1.0],
[1809, 1.1110606, 0.0019304405196777848, 1.0],
[1712, -0.5525154, 0.0051788902660723345, 1.0],
[1754, 0.5201581, 0.005691734062127954, 1.0],
[948, 1.6390722, 0.006622111055981219, 1.0],
[1810, 0.78618884, 0.007055917428377063, 1.0],
[779, 1.5241305, 0.007202934422407284, 1.0],
[1575, 1.0317602, 0.007830310753043345, 1.0],
[576, 0.97873515, 0.008272092578813124, 1.0],
[538, 0.89114505, 0.01062259019889307, 1.0],
[436, 0.3119122, 0.01127515110543434, 1.0]
]
self.compare_diffexp_results(results, expects)
self.compare_diffexp_results(results['positive'], positive_expects)
self.compare_diffexp_results(results['negative'], negative_expects)
def get_X_col(self, adaptor, cols):
varmask = np.zeros(adaptor.get_shape()[1], dtype=bool)
@@ -80,6 +97,7 @@ class DiffExpTest(unittest.TestCase):
self.check_1_10_2_10(results)
# run it directly
results = diffexp_ttest(adaptor, maskA, maskB, 10)
self.check_1_10_2_10(results)
@@ -128,15 +146,22 @@ class DiffExpTest(unittest.TestCase):
diffexp_results_sparse = diffexp_cxg.diffexp_ttest(adaptor_sparse, maskA, maskB, 10)
diffexp_results_dense = diffexp_cxg.diffexp_ttest(adaptor_dense, maskA, maskB, 10)
self.compare_diffexp_results(diffexp_results_anndata, diffexp_results_sparse)
self.compare_diffexp_results(diffexp_results_anndata, diffexp_results_dense)
self.compare_diffexp_results(diffexp_results_anndata['positive'], diffexp_results_sparse['positive'])
self.compare_diffexp_results(diffexp_results_anndata['negative'], diffexp_results_sparse['negative'])
self.compare_diffexp_results(diffexp_results_anndata['positive'], diffexp_results_dense['positive'])
self.compare_diffexp_results(diffexp_results_anndata['negative'], diffexp_results_dense['negative'])
topcols_pos = np.array([x[0] for x in diffexp_results_anndata['positive']])
topcols_neg = np.array([x[0] for x in diffexp_results_anndata['negative']])
topcols = np.concatenate((topcols_pos, topcols_neg))
topcols = np.array([x[0] for x in diffexp_results_anndata])
cols_anndata = self.get_X_col(adaptor_anndata, topcols)
cols_sparse = self.get_X_col(adaptor_sparse, topcols)
cols_dense = self.get_X_col(adaptor_dense, topcols)
assert cols_anndata.shape[0] == adaptor_sparse.get_shape()[0]
assert cols_anndata.shape[1] == len(diffexp_results_anndata)
assert cols_anndata.shape[1] == len(diffexp_results_anndata['positive']) + len(diffexp_results_anndata['negative'])
def convert(mat, cols):
return decode_matrix_fbs(encode_matrix_fbs(mat, col_idx=cols)).to_numpy()

6
backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor.py
View File

@@ -152,9 +152,11 @@ class AdaptorTest(unittest.TestCase):
f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
self.assertEqual(len(result), 10)
self.assertEqual(len(result['positive']), 10)
self.assertEqual(len(result['negative']), 10)
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
self.assertEqual(len(result), 20)
self.assertEqual(len(result['positive']), 20)
self.assertEqual(len(result['negative']), 20)
def test_data_frame(self):
f1 = {"var": {"index": [[0, 10]]}}

9
backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor_data_load.py
View File

@@ -30,10 +30,15 @@ class DataLoadAdaptorTest(unittest.TestCase):
def test_diffexp_topN(self):
f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
self.assertEqual(len(result), 10)
self.assertEqual(len(result['positive']), 10)
self.assertEqual(len(result['negative']), 10)
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
self.assertEqual(len(result), 20)
self.assertEqual(len(result['positive']), 20)
self.assertEqual(len(result['negative']), 20)
class DataLocatorAdaptorTest(unittest.TestCase):

2
backend/test/test_server/performance/run_diffexp.py
View File

@@ -4,7 +4,7 @@ import random
import time
import numpy as np
import backend.server.compute.diffexp_generic as diffexp_generic
import backend.common.compute.diffexp_generic as diffexp_generic
from backend.server.common.config.app_config import AppConfig
from backend.server.data_common.matrix_loader import MatrixDataLoader

6
backend/test/test_server/unit/common/test_api.py
View File

@@ -414,7 +414,8 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data), 7)
self.assertEqual(len(result_data['positive']), 7)
self.assertEqual(len(result_data['negative']), 7)
def test_diff_exp_indices(self):
endpoint = "diffexp/obs"
@@ -429,7 +430,8 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data), 10)
self.assertEqual(len(result_data['positive']), 10)
self.assertEqual(len(result_data['negative']), 10)
def test_get_summaryvar(self):
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]

52
backend/test/test_server/unit/compute/test_diffexp_cxg.py → backend/test/test_server/unit/compute/test_diffexp_h5ad.py
View File

@@ -2,13 +2,14 @@ import unittest
import numpy as np
from backend.common.compute import diffexp_generic
from backend.server.data_common.matrix_loader import MatrixDataLoader
from backend.test.test_server.unit import app_config
from backend.test import PROJECT_ROOT
class DiffExpTest(unittest.TestCase):
"""Tests the diffexp returns the expected results for one test case, using different
"""Tests the diffexp returns the expected results for one test case, using the h5ad
adaptor types and different algorithms."""
def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}):
@@ -35,19 +36,34 @@ class DiffExpTest(unittest.TestCase):
def check_1_10_2_10(self, results):
"""Checks the results for a specific set of rows selections"""
expects = [
positive_expects = [
[1712, -0.5525154, 0.0051788902660723345, 1.0],
[1575, 1.0317602, 0.007830310753043345, 1.0],
[693, 0.4703904, 0.008715846769131548, 1.0],
[916, 0.9567287, 0.009080596532247588, 1.0],
[77, 0.02665649, 0.010070392939027756, 1.0],
[782, -1.0981874, 0.010161745218916036, 1.0],
[913, 0.5683986, 0.010782030711612685, 1.0],
[910, 0.83164597, 0.014596411069229197, 1.0],
[1727, 0.4127781, 0.015168372104237176, 1.0],
[1443, -0.8241895, 0.015337080567465522, 1.0]
]
negative_expects = [
[956, 0.016060986, 0.0008649321884808977, 1.0],
[1124, 0.96602094, 0.0011717216548271284, 1.0],
[1809, 1.1110606, 0.0019304405196777848, 1.0],
[1712, -0.5525154, 0.0051788902660723345, 1.0],
[1754, 0.5201581, 0.005691734062127954, 1.0],
[948, 1.6390722, 0.006622111055981219, 1.0],
[1810, 0.78618884, 0.007055917428377063, 1.0],
[779, 1.5241305, 0.007202934422407284, 1.0],
[1575, 1.0317602, 0.007830310753043345, 1.0],
[576, 0.97873515, 0.008272092578813124, 1.0],
[538, 0.89114505, 0.01062259019889307, 1.0],
[436, 0.3119122, 0.01127515110543434, 1.0]
]
self.compare_diffexp_results(results, expects)
self.compare_diffexp_results(results["positive"], positive_expects)
self.compare_diffexp_results(results["negative"], negative_expects)
def get_X_col(self, adaptor, cols):
varmask = np.zeros(adaptor.get_shape()[1], dtype=bool)
@@ -61,3 +77,29 @@ class DiffExpTest(unittest.TestCase):
maskB = self.get_mask(adaptor, 2, 10)
results = adaptor.compute_diffexp_ttest(maskA, maskB, 10)
self.check_1_10_2_10(results)
def test_h5ad_default(self):
"""Test a h5ad adaptor with its default diffexp algorithm (diffexp_cxg)"""
adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
maskA = self.get_mask(adaptor, 1, 10)
maskB = self.get_mask(adaptor, 2, 10)
# run it through the adaptor
results = adaptor.compute_diffexp_ttest(maskA, maskB, 10)
self.check_1_10_2_10(results)
# run it directly
results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB, 10)
self.check_1_10_2_10(results)
def test_h5ad_generic(self):
"""Test a h5ad adaptor with the generic adaptor"""
adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
maskA = self.get_mask(adaptor, 1, 10)
maskB = self.get_mask(adaptor, 2, 10)
# run it directly
results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB, 10)
self.check_1_10_2_10(results)

7
backend/test/test_server/unit/data_anndata/test_anndata_adaptor.py
View File

@@ -153,9 +153,12 @@ class AdaptorTest(unittest.TestCase):
f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
self.assertEqual(len(result), 10)
self.assertEqual(len(result['positive']), 10)
self.assertEqual(len(result['negative']), 10)
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
self.assertEqual(len(result), 20)
self.assertEqual(len(result['positive']), 20)
self.assertEqual(len(result['negative']), 20)
def test_data_frame(self):
f1 = {"var": {"index": [[0, 10]]}}

6
backend/test/test_server/unit/data_anndata/test_anndata_adaptor_data_load.py
View File

@@ -31,9 +31,11 @@ class DataLoadAdaptorTest(unittest.TestCase):
f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
self.assertEqual(len(result), 10)
self.assertEqual(len(result['positive']), 10)
self.assertEqual(len(result['negative']), 10)
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
self.assertEqual(len(result), 20)
self.assertEqual(len(result['positive']), 20)
self.assertEqual(len(result['negative']), 20)
class DataLocatorAdaptorTest(unittest.TestCase):

13
client/src/actions/index.js
View File

@@ -211,15 +211,18 @@ const requestDifferentialExpression = (set1, set2, num_genes = 50) => async (
const response = await res.json();
const varIndex = await annoMatrix.fetch("var", varIndexName);
const data = response.map((v) => [
varIndex.at(v[0], varIndexName),
...v.slice(1),
]);
const diffexpLists = { negative: [], positive: [] };
for (const polarity of Object.keys(diffexpLists)) {
diffexpLists[polarity] = response[polarity].map((v) => [
varIndex.at(v[0], varIndexName),
...v.slice(1),
]);
}
/* then send the success case action through */
return dispatch({
type: "request differential expression success",
data,
data: diffexpLists,
});
} catch (error) {
return dispatch({

8
client/src/components/brushableHistogram/histogram.js
View File

@@ -26,7 +26,13 @@ const Histogram = ({
/*
Create the d3 histogram
*/
const { marginLeft, marginRight, marginBottom, marginTop } = margin;
// This is just a constant that's flipped by parent's `mini` boolean
const {
LEFT: marginLeft,
RIGHT: marginRight,
BOTTOM: marginBottom,
TOP: marginTop,
} = margin;
const { x, y, bins, binStart, binEnd, binWidth } = histogram;
const svg = d3.select(svgRef.current);
const binPadding = mini ? 0 : -1;

100
client/src/components/brushableHistogram/index.js
View File

@@ -13,6 +13,23 @@ import HistogramFooter from "./footer";
import StillLoading from "./loading";
import ErrorLoading from "./error";
const MARGIN = {
LEFT: 10, // Space for 0 tick label on X axis
RIGHT: 54, // space for Y axis & labels
BOTTOM: 25, // space for X axis & labels
TOP: 3,
};
const WIDTH = 340 - MARGIN.LEFT - MARGIN.RIGHT;
const HEIGHT = 135 - MARGIN.TOP - MARGIN.BOTTOM;
const MARGIN_MINI = {
LEFT: 0, // Space for 0 tick label on X axis
RIGHT: 0, // space for Y axis & labels
BOTTOM: 0, // space for X axis & labels
TOP: 0,
};
const WIDTH_MINI = 120 - MARGIN_MINI.LEFT - MARGIN_MINI.RIGHT;
const HEIGHT_MINI = 15 - MARGIN_MINI.TOP - MARGIN_MINI.BOTTOM;
@connect((state, ownProps) => {
const { isObs, isUserDefined, isGeneSetSummary, field } = ownProps;
const myName = makeContinuousDimensionName(
@@ -44,34 +61,6 @@ class HistogramBrush extends React.PureComponent {
}
});
constructor(props) {
super(props);
const marginLeft = 10; // Space for 0 tick label on X axis
const marginRight = 54; // space for Y axis & labels
const marginBottom = 25; // space for X axis & labels
const marginTop = 3;
this.state = {
margin: {
marginLeft,
marginRight,
marginBottom,
marginTop,
},
width: 340 - marginLeft - marginRight,
height: 135 - marginTop - marginBottom,
marginMini: {
marginLeft: 0, // Space for 0 tick label on X axis
marginRight: 0, // space for Y axis & labels
marginBottom: 0, // space for X axis & labels
marginTop: 0,
},
widthMini: 120,
heightMini: 15,
};
}
onBrush = (selection, x, eventType) => {
const type = `continuous metadata histogram ${eventType}`;
return () => {
@@ -210,15 +199,8 @@ class HistogramBrush extends React.PureComponent {
};
fetchAsyncProps = async () => {
const { annoMatrix } = this.props;
const {
margin,
width,
height,
marginMini,
widthMini,
heightMini,
} = this.state;
const { annoMatrix, width } = this.props;
const { isClipped } = annoMatrix;
const query = this.createQuery();
@@ -246,12 +228,17 @@ class HistogramBrush extends React.PureComponent {
: globals.blue,
];
const histogram = this.calcHistogramCache(column, margin, width, height);
const histogram = this.calcHistogramCache(
column,
MARGIN,
width || WIDTH,
HEIGHT
);
const miniHistogram = this.calcHistogramCache(
column,
marginMini,
widthMini,
heightMini
MARGIN_MINI,
width || WIDTH_MINI,
HEIGHT_MINI
);
const isSingleValue = summary.min === summary.max;
@@ -275,7 +262,7 @@ class HistogramBrush extends React.PureComponent {
};
// eslint-disable-next-line class-methods-use-this -- instance method allows for memoization per annotation
calcHistogramCache(col, margin, width, height) {
calcHistogramCache(col, newMargin, newWidth, newHeight) {
/*
recalculate expensive stuff, notably bins, summaries, etc.
*/
@@ -283,7 +270,10 @@ class HistogramBrush extends React.PureComponent {
const summary = col.summarize(); /* this is memoized, so it's free the second time you call it */
const { min: domainMin, max: domainMax } = summary;
const numBins = 40;
const { marginTop, marginLeft } = margin; /* changes with mini */
const {
TOP: topMargin,
LEFT: leftMargin,
} = newMargin; /* changes with mini */
histogramCache.domain = [
domainMin,
@@ -293,7 +283,7 @@ class HistogramBrush extends React.PureComponent {
histogramCache.x = d3
.scaleLinear()
.domain([domainMin, domainMax])
.range([marginLeft, marginLeft + width]);
.range([leftMargin, leftMargin + newWidth]);
histogramCache.bins = histogramContinuous(col, numBins, [
domainMin,
@@ -310,7 +300,7 @@ class HistogramBrush extends React.PureComponent {
histogramCache.y = d3
.scaleLinear()
.domain([0, yMax])
.range([marginTop + height, marginTop]);
.range([topMargin + newHeight, topMargin]);
return histogramCache;
}
@@ -367,14 +357,12 @@ class HistogramBrush extends React.PureComponent {
mini,
setGenes,
} = this.props;
const {
margin,
width,
height,
marginMini,
widthMini,
heightMini,
} = this.state;
let { width } = this.props;
if (!width) {
width = mini ? WIDTH_MINI : WIDTH;
}
const fieldForId = field.replace(/\s/g, "_");
const showScatterPlot = isUserDefined;
@@ -432,11 +420,11 @@ class HistogramBrush extends React.PureComponent {
histogram={
mini ? asyncProps.miniHistogram : asyncProps.histogram
}
width={mini ? widthMini : width}
height={mini ? heightMini : height}
width={width}
height={mini ? HEIGHT_MINI : HEIGHT}
onBrush={this.onBrush}
onBrushEnd={this.onBrushEnd}
margin={mini ? marginMini : margin}
margin={mini ? MARGIN_MINI : MARGIN}
isColorBy={isColorAccessor}
selectionRange={continuousSelectionRange}
mini={mini}

32
client/src/components/geneExpression/gene.js
View File

@@ -1,13 +1,14 @@
import React from "react";
import { connect } from "react-redux";
import { AnchorButton, Icon } from "@blueprintjs/core";
import { Button, Icon } from "@blueprintjs/core";
import Truncate from "../util/truncate";
import HistogramBrush from "../brushableHistogram";
import * as globals from "../../globals";
import actions from "../../actions";
const MINI_HISTOGRAM_WIDTH = 110;
@connect((state, ownProps) => {
const { gene } = ownProps;
@@ -65,7 +66,7 @@ class Gene extends React.Component {
isScatterplotYYaccessor,
} = this.props;
const { geneIsExpanded } = this.state;
const genesetNameLengthVisible = 310; /* this magic number determines how much of a long geneset name we see */
const geneSymbolWidth = 60 + (geneIsExpanded ? MINI_HISTOGRAM_WIDTH : 0);
return (
<div>
@@ -108,7 +109,8 @@ class Gene extends React.Component {
>
<span
style={{
width: globals.leftSidebarWidth - genesetNameLengthVisible,
width: geneSymbolWidth,
display: "inline-block",
}}
data-testid={`${gene}:gene-label`}
>
@@ -117,11 +119,16 @@ class Gene extends React.Component {
</Truncate>
</div>
{!geneIsExpanded ? (
<HistogramBrush isUserDefined field={gene} mini />
<HistogramBrush
isUserDefined
field={gene}
mini
width={MINI_HISTOGRAM_WIDTH}
/>
) : null}
</div>
<div style={{ flexShrink: 0, marginLeft: 2 }}>
<AnchorButton
<Button
minimal
small
data-testid={`delete-from-geneset-${gene}`}
@@ -130,8 +137,7 @@ class Gene extends React.Component {
style={{ fontWeight: 700, marginRight: 2 }}
icon={<Icon icon="trash" iconSize={10} />}
/>
)
<AnchorButton
<Button
minimal
small
data-testid={`plot-x-${gene}`}
@@ -141,8 +147,8 @@ class Gene extends React.Component {
style={{ fontWeight: 700, marginRight: 2 }}
>
x
</AnchorButton>
<AnchorButton
</Button>
<Button
minimal
small
data-testid={`plot-y-${gene}`}
@@ -152,8 +158,8 @@ class Gene extends React.Component {
style={{ fontWeight: 700, marginRight: 2 }}
>
y
</AnchorButton>
<AnchorButton
</Button>
<Button
minimal
small
data-testclass="maximize"
@@ -164,7 +170,7 @@ class Gene extends React.Component {
icon={<Icon icon="maximize" iconSize={10} />}
style={{ marginRight: 2 }}
/>
<AnchorButton
<Button
minimal
small
data-testclass="colorby"

56
client/src/reducers/genesets.js
View File

@@ -96,12 +96,17 @@ const GeneSets = (
if (state.genesets.has(genesetName))
throw new Error("geneset: create -- name already defined.");
const genesets = new Map(state.genesets); // clone
genesets.set(genesetName, {
genesetName,
genesetDescription,
genes: new Map(),
});
const genesets = new Map([
[
genesetName,
{
genesetName,
genesetDescription,
genes: new Map(),
},
],
...state.genesets,
]); // clone and add new geneset to beginning
return {
...state,
@@ -357,23 +362,34 @@ const GeneSets = (
case "request differential expression success": {
const { data } = action;
const genes = new Map(
data.map((diffExpGene) => [
diffExpGene[0],
const dateString = new Date().toLocaleString();
const genesetNames = {
positive: `Pop1 high (${dateString})`,
negative: `Pop2 high (${dateString})`,
};
const diffExpGeneSets = [];
for (const polarity of Object.keys(genesetNames)) {
const genes = new Map(
data[polarity].map((diffExpGene) => [
diffExpGene[0],
{
geneSymbol: diffExpGene[0],
},
])
);
diffExpGeneSets.push([
genesetNames[polarity],
{
geneSymbol: diffExpGene[0],
genesetName: genesetNames[polarity],
genesetDescription: "",
genes,
},
])
);
]);
}
const genesetName = `DiffExp Set (${new Date().toLocaleString()})`;
const genesets = new Map(state.genesets); // clone
genesets.set(genesetName, {
genesetName,
genesetDescription: "",
genes,
});
const genesets = new Map([...diffExpGeneSets, ...state.genesets]); // clone
return {
...state,