fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-30 20:38:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: diffexp returns two genesets (#2230)

Browse Source 
* feat: return two lists for diffexp (#2221)

* sp

* split out derive sort order, tests passing

* sp

* return diff exp results in two lists

* update

* copy implementation over to desktop

* add tests for two lists

* small fixes to complete backend implementation

* accept new diffexp response

* map diff exp response to genesets

* delete )

* name diffexp genesets with population names

* take constants out of state and allow width prop to override

* shorten mini-histo properly truncate and resize depending on expansion

* prepend new genesets

* rename data within diffexp action

* backend

* move diffexp ttest to common code module, update tests

* update for unit tests

* reference actual var

Co-authored-by: Madison Dunitz <madison.dunitz@chanzuckerberg.com>
Co-authored-by: Madison Dunitz <dunitzm@gmail.com>
This commit is contained in:
Severiano Badajoz
2021-06-08 21:02:19 +00:00
committed by GitHub
co-authored by Madison Dunitz Madison Dunitz
parent 7ed53c0f5b
commit 28b526b3fc
25 changed files with 268 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/common/compute/__init__.py
View File
backend/czi_hosted/compute/diffexp_generic.py → backend/common/compute/diffexp_generic.py
+17 -9
View File
@@ -25,7 +25,8 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
:param maskB: observation selection mask for set 2
:param top_n: number of variables to return stats for
:param diffexp_lfc_cutoff: minimum
:return: for top N genes, [ varindex, logfoldchange, pval, pval_adj ]
absolute value returning [ varindex, logfoldchange, pval, pval_adj ] for top N genes
:return: for top N genes, {"positive": for top N genes, [ varindex, logfoldchange, pval, pval_adj ], "negative": for top N genes, [ varindex, logfoldchange, pval, pval_adj ]}
"""
dataA = adaptor.get_X_array(maskA, None)
@@ -66,24 +67,27 @@ def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp
# logfoldchanges: log2(meanA / meanB)
logfoldchanges = np.log2(np.abs((meanA + 1e-9) / (meanB + 1e-9)))
stats_to_sort = tscores
# find all with lfc > cutoff
lfc_above_cutoff_idx = np.nonzero(np.abs(logfoldchanges) > diffexp_lfc_cutoff)[0]
stats_to_sort = np.abs(tscores)
# derive sort order
if lfc_above_cutoff_idx.shape[0] > top_n:
if lfc_above_cutoff_idx.shape[0] > top_n*2:
# partition top N
rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], -top_n)[-top_n:]
t_partition = lfc_above_cutoff_idx[rel_t_partition]
rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], (top_n, -top_n))
rel_t_partition_top_n = np.concatenate((rel_t_partition[-top_n:], rel_t_partition[:top_n]))
t_partition = lfc_above_cutoff_idx[rel_t_partition_top_n]
# sort the top N partition
rel_sort_order = np.argsort(stats_to_sort[t_partition])[::-1]
sort_order = t_partition[rel_sort_order]
else:
# partition and sort top N, ignoring lfc cutoff
partition = np.argpartition(stats_to_sort, -top_n)[-top_n:]
rel_sort_order = np.argsort(stats_to_sort[partition])[::-1]
partition = np.argpartition(stats_to_sort, (top_n, -top_n))
partition_top_n = np.concatenate((partition[-top_n:], partition[:top_n]))
rel_sort_order = np.argsort(stats_to_sort[partition_top_n])[::-1]
indices = np.indices(stats_to_sort.shape)[0]
sort_order = indices[partition][rel_sort_order]
sort_order = indices[partition_top_n][rel_sort_order]
# top n slice based upon sort order
logfoldchanges_top_n = logfoldchanges[sort_order]
@@ -91,7 +95,11 @@ def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp
pvals_adj_top_n = pvals_adj[sort_order]
# varIndex, logfoldchange, pval, pval_adj
result = [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n)]
result = {"positive": [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in
range(top_n)],
"negative": [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in
range(-1, -1 - top_n, -1)], }
return result
backend/czi_hosted/common/rest.py
-1
View File
@@ -260,7 +260,6 @@ def diffexp_obs_post(request, data_adaptor):
try:
# TODO: implement varfilter mode
mode = DiffExpMode(args["mode"])
if mode == DiffExpMode.VAR_FILTER or "varFilter" in args:
return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, "varFilter not enabled")
backend/czi_hosted/compute/diffexp_cxg.py
+9 -9
View File
@@ -4,7 +4,7 @@ import numpy as np
from numba import jit
from backend.czi_hosted.data_cxg.cxg_util import pack_selector_from_indices
from backend.czi_hosted.compute.diffexp_generic import diffexp_ttest_from_mean_var, mean_var_n
from backend.common.compute.diffexp_generic import diffexp_ttest_from_mean_var, mean_var_n
from backend.common.errors import ComputeError
"""
@@ -115,14 +115,14 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
meanB += X_col_shift
r = diffexp_ttest_from_mean_var(
meanA.astype(dtype),
varA.astype(dtype),
nA,
meanB.astype(dtype),
varB.astype(dtype),
nB,
top_n,
diffexp_lfc_cutoff,
meanA=meanA.astype(dtype),
varA=varA.astype(dtype),
nA=nA,
meanB=meanB.astype(dtype),
varB=varB.astype(dtype),
nB=nB,
top_n=top_n,
diffexp_lfc_cutoff=diffexp_lfc_cutoff
)
return r
backend/czi_hosted/data_anndata/anndata_adaptor.py
+1 -1
View File
@@ -8,7 +8,7 @@ from pandas.core.dtypes.dtypes import CategoricalDtype
from scipy import sparse
from server_timing import Timing as ServerTiming
import backend.czi_hosted.compute.diffexp_generic as diffexp_generic
import backend.common.compute.diffexp_generic as diffexp_generic
from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
from backend.common.constants import Axis, MAX_LAYOUTS
from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata
backend/czi_hosted/data_common/data_adaptor.py
+4 -3
View File
@@ -163,7 +163,7 @@ class DataAdaptor(metaclass=ABCMeta):
mask = np.zeros((count,), dtype=np.bool)
for i in filter:
if type(i) == list:
mask[i[0] : i[1]] = True
mask[i[0]: i[1]] = True
else:
mask[i] = True
return mask
@@ -321,11 +321,12 @@ class DataAdaptor(metaclass=ABCMeta):
top_n = self.dataset_config.diffexp__top_n
if self.server_config.exceeds_limit(
"diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
"diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
):
raise ExceedsLimitError("Diffexp request exceeds max cell count limit")
result = self.compute_diffexp_ttest(obs_mask_A, obs_mask_B, top_n, self.dataset_config.diffexp__lfc_cutoff)
result = self.compute_diffexp_ttest(
maskA=obs_mask_A, maskB=obs_mask_B, top_n=top_n, lfc_cutoff=self.dataset_config.diffexp__lfc_cutoff)
try:
return jsonify_numpy(result)
backend/czi_hosted/data_cxg/cxg_adaptor.py
+2 -1
View File
@@ -207,7 +207,8 @@ class CxgAdaptor(DataAdaptor):
top_n = self.dataset_config.diffexp__top_n
if lfc_cutoff is None:
lfc_cutoff = self.dataset_config.diffexp__lfc_cutoff
return diffexp_cxg.diffexp_ttest(self, maskA, maskB, top_n, lfc_cutoff)
return diffexp_cxg.diffexp_ttest(
adaptor=self, maskA=maskA, maskB=maskB, top_n=top_n, diffexp_lfc_cutoff=lfc_cutoff)
def get_colors(self):
if self.cxg_version == "0.0":
backend/server/compute/diffexp_generic.py
-134
View File
@@ -1,134 +0,0 @@
import numpy as np
from scipy import sparse, stats
def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
"""
Return differential expression statistics for top N variables.
Algorithm:
- compute log fold change (log2(meanA/meanB))
- compute Welch's t-test statistic and pvalue (w/ Bonferroni correction)
- return top N abs(logfoldchange) where lfc > diffexp_lfc_cutoff
If there are not N which meet criteria, augment by removing the logfoldchange
threshold requirement.
Notes on alogrithm:
- Welch's ttest provides basic statistics test.
https://en.wikipedia.org/wiki/Welch%27s_t-test
- p-values adjusted with Bonferroni correction.
https://en.wikipedia.org/wiki/Bonferroni_correction
:param adaptor: DataAdaptor instance
:param maskA: observation selection mask for set 1
:param maskB: observation selection mask for set 2
:param top_n: number of variables to return stats for
:param diffexp_lfc_cutoff: minimum
:return: for top N genes, [ varindex, logfoldchange, pval, pval_adj ]
"""
dataA = adaptor.get_X_array(maskA, None)
dataB = adaptor.get_X_array(maskB, None)
# mean, variance, N - calculate for both selections
meanA, vA, nA = mean_var_n(dataA)
meanB, vB, nB = mean_var_n(dataB)
res = diffexp_ttest_from_mean_var(meanA, vA, nA, meanB, vB, nB, top_n, diffexp_lfc_cutoff)
return res
def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp_lfc_cutoff):
n_var = meanA.shape[0]
top_n = min(top_n, n_var)
# variance / N
vnA = varA / min(nA, nB) # overestimate variance, would normally be nA
vnB = varB / min(nA, nB) # overestimate variance, would normally be nB
sum_vn = vnA + vnB
# degrees of freedom for Welch's t-test
with np.errstate(divide="ignore", invalid="ignore"):
dof = sum_vn ** 2 / (vnA ** 2 / (nA - 1) + vnB ** 2 / (nB - 1))
dof[np.isnan(dof)] = 1
# Welch's t-test score calculation
with np.errstate(divide="ignore", invalid="ignore"):
tscores = (meanA - meanB) / np.sqrt(sum_vn)
tscores[np.isnan(tscores)] = 0
# p-value
pvals = stats.t.sf(np.abs(tscores), dof) * 2
pvals_adj = pvals * n_var
pvals_adj[pvals_adj > 1] = 1 # cap adjusted p-value at 1
# logfoldchanges: log2(meanA / meanB)
logfoldchanges = np.log2(np.abs((meanA + 1e-9) / (meanB + 1e-9)))
# find all with lfc > cutoff
lfc_above_cutoff_idx = np.nonzero(np.abs(logfoldchanges) > diffexp_lfc_cutoff)[0]
stats_to_sort = np.abs(tscores)
# derive sort order
if lfc_above_cutoff_idx.shape[0] > top_n:
# partition top N
rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], -top_n)[-top_n:]
t_partition = lfc_above_cutoff_idx[rel_t_partition]
# sort the top N partition
rel_sort_order = np.argsort(stats_to_sort[t_partition])[::-1]
sort_order = t_partition[rel_sort_order]
else:
# partition and sort top N, ignoring lfc cutoff
partition = np.argpartition(stats_to_sort, -top_n)[-top_n:]
rel_sort_order = np.argsort(stats_to_sort[partition])[::-1]
indices = np.indices(stats_to_sort.shape)[0]
sort_order = indices[partition][rel_sort_order]
# top n slice based upon sort order
logfoldchanges_top_n = logfoldchanges[sort_order]
pvals_top_n = pvals[sort_order]
pvals_adj_top_n = pvals_adj[sort_order]
# varIndex, logfoldchange, pval, pval_adj
result = [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n)]
return result
# Convenience function which handles sparse data
def mean_var_n(X):
"""
Two-pass variance calculation. Numerically (more) stable
than naive methods (and same method used by numpy.var())
https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Two-pass
"""
# fp_err_occurred is a flag indicating that a floating point error
# occured somewhere in our compute. Used to trigger non-finite
# number handling.
fp_err_occurred = False
def fp_err_set(err, flag):
nonlocal fp_err_occurred
fp_err_occurred = True
with np.errstate(divide="call", invalid="call", call=fp_err_set):
n = X.shape[0]
if sparse.issparse(X):
mean = X.mean(axis=0).A1
dfm = X - mean
sumsq = np.sum(np.multiply(dfm, dfm), axis=0).A1
v = sumsq / (n - 1)
else:
mean = X.mean(axis=0)
dfm = X - mean
sumsq = np.sum(np.multiply(dfm, dfm), axis=0)
v = sumsq / (n - 1)
if fp_err_occurred:
mean[np.isfinite(mean) == False] = 0 # noqa: E712
v[np.isfinite(v) == False] = 0 # noqa: E712
else:
mean[np.isnan(mean)] = 0
v[np.isnan(v)] = 0
return mean, v, n
backend/server/data_anndata/anndata_adaptor.py
+1 -1
View File
@@ -8,7 +8,7 @@ from pandas.core.dtypes.dtypes import CategoricalDtype
from scipy import sparse
from server_timing import Timing as ServerTiming
import backend.server.compute.diffexp_generic as diffexp_generic
import backend.common.compute.diffexp_generic as diffexp_generic
from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
from backend.common.constants import Axis, MAX_LAYOUTS
from backend.server.common.corpora import corpora_get_props_from_anndata
backend/server/data_common/data_adaptor.py
+7 -2
View File
@@ -68,7 +68,7 @@ class DataAdaptor(metaclass=ABCMeta):
@abstractmethod
def compute_embedding(self, method, filter):
"""compute a new embedding on the specified obs subset, and return the embedding schema. """
"""compute a new embedding on the specified obs subset, and return the embedding schema."""
pass
@abstractmethod
@@ -324,7 +324,12 @@ class DataAdaptor(metaclass=ABCMeta):
):
raise ExceedsLimitError("Diffexp request exceeds max cell count limit")
result = self.compute_diffexp_ttest(obs_mask_A, obs_mask_B, top_n, self.dataset_config.diffexp__lfc_cutoff)
result = self.compute_diffexp_ttest(
maskA=obs_mask_A,
maskB=obs_mask_B,
top_n=top_n,
lfc_cutoff=self.dataset_config.diffexp__lfc_cutoff,
)
try:
return jsonify_numpy(result)
backend/test/test_czi_hosted/performance/run_diffexp.py
+2 -1
View File
@@ -5,7 +5,8 @@ import time
import numpy as np
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.czi_hosted.compute import diffexp_generic, diffexp_cxg
from backend.czi_hosted.compute import diffexp_cxg
from backend.common.compute import diffexp_generic
from backend.czi_hosted.data_common.matrix_loader import MatrixDataLoader
from backend.czi_hosted.data_cxg.cxg_adaptor import CxgAdaptor
backend/test/test_czi_hosted/unit/common/test_api.py
+5 -2
View File
@@ -158,7 +158,8 @@ class EndPoints(object):
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data), 7)
self.assertEqual(len(result_data['positive']), 7)
self.assertEqual(len(result_data['negative']), 7)
def test_diff_exp_indices(self):
endpoint = "diffexp/obs"
@@ -173,7 +174,8 @@ class EndPoints(object):
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data), 10)
self.assertEqual(len(result_data['positive']), 10)
self.assertEqual(len(result_data['negative']), 10)
def test_get_annotations_var_fbs(self):
endpoint = "annotations/var"
@@ -382,6 +384,7 @@ class EndPoints(object):
query_hash = hashlib.sha1(query.encode()).hexdigest()
url = f"{self.URL_BASE}{endpoint}?key={query_hash}"
result = self.session.post(url, headers=headers, data=query)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
backend/test/test_czi_hosted/unit/compute/test_diffexp_cxg.py
+34 -9
View File
@@ -4,7 +4,8 @@ import unittest
import numpy as np
from backend.czi_hosted.compute import diffexp_generic, diffexp_cxg
from backend.czi_hosted.compute import diffexp_cxg
from backend.common.compute import diffexp_generic
from backend.czi_hosted.compute.diffexp_cxg import diffexp_ttest
from backend.czi_hosted.converters.h5ad_data_file import H5ADDataFile
from backend.common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
@@ -40,21 +41,37 @@ class DiffExpTest(unittest.TestCase):
self.assertTrue(np.isclose(result[2], expect[2], 1e-6, 1e-4))
self.assertTrue(np.isclose(result[3], expect[3], 1e-6, 1e-4))
def check_1_10_2_10(self, results):
"""Checks the results for a specific set of rows selections"""
expects = [
positive_expects = [
[1712, -0.5525154, 0.0051788902660723345, 1.0],
[1575, 1.0317602, 0.007830310753043345, 1.0],
[693, 0.4703904, 0.008715846769131548, 1.0],
[916, 0.9567287, 0.009080596532247588, 1.0],
[77, 0.02665649, 0.010070392939027756, 1.0],
[782, -1.0981874, 0.010161745218916036, 1.0],
[913, 0.5683986, 0.010782030711612685, 1.0],
[910, 0.83164597, 0.014596411069229197, 1.0],
[1727, 0.4127781, 0.015168372104237176, 1.0],
[1443, -0.8241895, 0.015337080567465522, 1.0]
]
negative_expects = [
[956, 0.016060986, 0.0008649321884808977, 1.0],
[1124, 0.96602094, 0.0011717216548271284, 1.0],
[1809, 1.1110606, 0.0019304405196777848, 1.0],
[1712, -0.5525154, 0.0051788902660723345, 1.0],
[1754, 0.5201581, 0.005691734062127954, 1.0],
[948, 1.6390722, 0.006622111055981219, 1.0],
[1810, 0.78618884, 0.007055917428377063, 1.0],
[779, 1.5241305, 0.007202934422407284, 1.0],
[1575, 1.0317602, 0.007830310753043345, 1.0],
[576, 0.97873515, 0.008272092578813124, 1.0],
[538, 0.89114505, 0.01062259019889307, 1.0],
[436, 0.3119122, 0.01127515110543434, 1.0]
]
self.compare_diffexp_results(results, expects)
self.compare_diffexp_results(results['positive'], positive_expects)
self.compare_diffexp_results(results['negative'], negative_expects)
def get_X_col(self, adaptor, cols):
varmask = np.zeros(adaptor.get_shape()[1], dtype=bool)
@@ -80,6 +97,7 @@ class DiffExpTest(unittest.TestCase):
self.check_1_10_2_10(results)
# run it directly
results = diffexp_ttest(adaptor, maskA, maskB, 10)
self.check_1_10_2_10(results)
@@ -128,15 +146,22 @@ class DiffExpTest(unittest.TestCase):
diffexp_results_sparse = diffexp_cxg.diffexp_ttest(adaptor_sparse, maskA, maskB, 10)
diffexp_results_dense = diffexp_cxg.diffexp_ttest(adaptor_dense, maskA, maskB, 10)
self.compare_diffexp_results(diffexp_results_anndata, diffexp_results_sparse)
self.compare_diffexp_results(diffexp_results_anndata, diffexp_results_dense)
self.compare_diffexp_results(diffexp_results_anndata['positive'], diffexp_results_sparse['positive'])
self.compare_diffexp_results(diffexp_results_anndata['negative'], diffexp_results_sparse['negative'])
self.compare_diffexp_results(diffexp_results_anndata['positive'], diffexp_results_dense['positive'])
self.compare_diffexp_results(diffexp_results_anndata['negative'], diffexp_results_dense['negative'])
topcols_pos = np.array([x[0] for x in diffexp_results_anndata['positive']])
topcols_neg = np.array([x[0] for x in diffexp_results_anndata['negative']])
topcols = np.concatenate((topcols_pos, topcols_neg))
topcols = np.array([x[0] for x in diffexp_results_anndata])
cols_anndata = self.get_X_col(adaptor_anndata, topcols)
cols_sparse = self.get_X_col(adaptor_sparse, topcols)
cols_dense = self.get_X_col(adaptor_dense, topcols)
assert cols_anndata.shape[0] == adaptor_sparse.get_shape()[0]
assert cols_anndata.shape[1] == len(diffexp_results_anndata)
assert cols_anndata.shape[1] == len(diffexp_results_anndata['positive']) + len(diffexp_results_anndata['negative'])
def convert(mat, cols):
return decode_matrix_fbs(encode_matrix_fbs(mat, col_idx=cols)).to_numpy()
backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor.py
+4 -2
View File
@@ -152,9 +152,11 @@ class AdaptorTest(unittest.TestCase):
f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
self.assertEqual(len(result), 10)
self.assertEqual(len(result['positive']), 10)
self.assertEqual(len(result['negative']), 10)
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
self.assertEqual(len(result), 20)
self.assertEqual(len(result['positive']), 20)
self.assertEqual(len(result['negative']), 20)
def test_data_frame(self):
f1 = {"var": {"index": [[0, 10]]}}
backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor_data_load.py
+7 -2
View File
@@ -30,10 +30,15 @@ class DataLoadAdaptorTest(unittest.TestCase):
def test_diffexp_topN(self):
f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
self.assertEqual(len(result), 10)
self.assertEqual(len(result['positive']), 10)
self.assertEqual(len(result['negative']), 10)
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
self.assertEqual(len(result), 20)
self.assertEqual(len(result['positive']), 20)
self.assertEqual(len(result['negative']), 20)
class DataLocatorAdaptorTest(unittest.TestCase):
backend/test/test_server/performance/run_diffexp.py
+1 -1
View File
@@ -4,7 +4,7 @@ import random
import time
import numpy as np
import backend.server.compute.diffexp_generic as diffexp_generic
import backend.common.compute.diffexp_generic as diffexp_generic
from backend.server.common.config.app_config import AppConfig
from backend.server.data_common.matrix_loader import MatrixDataLoader
backend/test/test_server/unit/common/test_api.py
+4 -2
View File
@@ -414,7 +414,8 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data), 7)
self.assertEqual(len(result_data['positive']), 7)
self.assertEqual(len(result_data['negative']), 7)
def test_diff_exp_indices(self):
endpoint = "diffexp/obs"
@@ -429,7 +430,8 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data), 10)
self.assertEqual(len(result_data['positive']), 10)
self.assertEqual(len(result_data['negative']), 10)
def test_get_summaryvar(self):
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
backend/test/test_server/unit/compute/test_diffexp_cxg.py → backend/test/test_server/unit/compute/test_diffexp_h5ad.py
+47 -5
View File
@@ -2,13 +2,14 @@ import unittest
import numpy as np
from backend.common.compute import diffexp_generic
from backend.server.data_common.matrix_loader import MatrixDataLoader
from backend.test.test_server.unit import app_config
from backend.test import PROJECT_ROOT
class DiffExpTest(unittest.TestCase):
"""Tests the diffexp returns the expected results for one test case, using different
"""Tests the diffexp returns the expected results for one test case, using the h5ad
adaptor types and different algorithms."""
def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}):
@@ -35,19 +36,34 @@ class DiffExpTest(unittest.TestCase):
def check_1_10_2_10(self, results):
"""Checks the results for a specific set of rows selections"""
expects = [
positive_expects = [
[1712, -0.5525154, 0.0051788902660723345, 1.0],
[1575, 1.0317602, 0.007830310753043345, 1.0],
[693, 0.4703904, 0.008715846769131548, 1.0],
[916, 0.9567287, 0.009080596532247588, 1.0],
[77, 0.02665649, 0.010070392939027756, 1.0],
[782, -1.0981874, 0.010161745218916036, 1.0],
[913, 0.5683986, 0.010782030711612685, 1.0],
[910, 0.83164597, 0.014596411069229197, 1.0],
[1727, 0.4127781, 0.015168372104237176, 1.0],
[1443, -0.8241895, 0.015337080567465522, 1.0]
]
negative_expects = [
[956, 0.016060986, 0.0008649321884808977, 1.0],
[1124, 0.96602094, 0.0011717216548271284, 1.0],
[1809, 1.1110606, 0.0019304405196777848, 1.0],
[1712, -0.5525154, 0.0051788902660723345, 1.0],
[1754, 0.5201581, 0.005691734062127954, 1.0],
[948, 1.6390722, 0.006622111055981219, 1.0],
[1810, 0.78618884, 0.007055917428377063, 1.0],
[779, 1.5241305, 0.007202934422407284, 1.0],
[1575, 1.0317602, 0.007830310753043345, 1.0],
[576, 0.97873515, 0.008272092578813124, 1.0],
[538, 0.89114505, 0.01062259019889307, 1.0],
[436, 0.3119122, 0.01127515110543434, 1.0]
]
self.compare_diffexp_results(results, expects)
self.compare_diffexp_results(results["positive"], positive_expects)
self.compare_diffexp_results(results["negative"], negative_expects)
def get_X_col(self, adaptor, cols):
varmask = np.zeros(adaptor.get_shape()[1], dtype=bool)
@@ -61,3 +77,29 @@ class DiffExpTest(unittest.TestCase):
maskB = self.get_mask(adaptor, 2, 10)
results = adaptor.compute_diffexp_ttest(maskA, maskB, 10)
self.check_1_10_2_10(results)
def test_h5ad_default(self):
"""Test a h5ad adaptor with its default diffexp algorithm (diffexp_cxg)"""
adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
maskA = self.get_mask(adaptor, 1, 10)
maskB = self.get_mask(adaptor, 2, 10)
# run it through the adaptor
results = adaptor.compute_diffexp_ttest(maskA, maskB, 10)
self.check_1_10_2_10(results)
# run it directly
results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB, 10)
self.check_1_10_2_10(results)
def test_h5ad_generic(self):
"""Test a h5ad adaptor with the generic adaptor"""
adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
maskA = self.get_mask(adaptor, 1, 10)
maskB = self.get_mask(adaptor, 2, 10)
# run it directly
results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB, 10)
self.check_1_10_2_10(results)
backend/test/test_server/unit/data_anndata/test_anndata_adaptor.py
+5 -2
View File
@@ -153,9 +153,12 @@ class AdaptorTest(unittest.TestCase):
f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
self.assertEqual(len(result), 10)
self.assertEqual(len(result['positive']), 10)
self.assertEqual(len(result['negative']), 10)
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
self.assertEqual(len(result), 20)
self.assertEqual(len(result['positive']), 20)
self.assertEqual(len(result['negative']), 20)
def test_data_frame(self):
f1 = {"var": {"index": [[0, 10]]}}
backend/test/test_server/unit/data_anndata/test_anndata_adaptor_data_load.py
+4 -2
View File
@@ -31,9 +31,11 @@ class DataLoadAdaptorTest(unittest.TestCase):
f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
self.assertEqual(len(result), 10)
self.assertEqual(len(result['positive']), 10)
self.assertEqual(len(result['negative']), 10)
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
self.assertEqual(len(result), 20)
self.assertEqual(len(result['positive']), 20)
self.assertEqual(len(result['negative']), 20)
class DataLocatorAdaptorTest(unittest.TestCase):