fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-10-01 18:48:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: diffexp returns two genesets (#2230)

Browse Source 
* feat: return two lists for diffexp (#2221)

* sp

* split out derive sort order, tests passing

* sp

* return diff exp results in two lists

* update

* copy implementation over to desktop

* add tests for two lists

* small fixes to complete backend implementation

* accept new diffexp response

* map diff exp response to genesets

* delete )

* name diffexp genesets with population names

* take constants out of state and allow width prop to override

* shorten mini-histo properly truncate and resize depending on expansion

* prepend new genesets

* rename data within diffexp action

* backend

* move diffexp ttest to common code module, update tests

* update for unit tests

* reference actual var

Co-authored-by: Madison Dunitz <madison.dunitz@chanzuckerberg.com>
Co-authored-by: Madison Dunitz <dunitzm@gmail.com>
This commit is contained in:
Severiano Badajoz
2021-06-08 21:02:19 +00:00
committed by GitHub
co-authored by Madison Dunitz Madison Dunitz
parent 7ed53c0f5b
commit 28b526b3fc
25 changed files with 268 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/test/test_czi_hosted/performance/run_diffexp.py
+2 -1
View File
@@ -5,7 +5,8 @@ import time
import numpy as np
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.czi_hosted.compute import diffexp_generic, diffexp_cxg
from backend.czi_hosted.compute import diffexp_cxg
from backend.common.compute import diffexp_generic
from backend.czi_hosted.data_common.matrix_loader import MatrixDataLoader
from backend.czi_hosted.data_cxg.cxg_adaptor import CxgAdaptor
backend/test/test_czi_hosted/unit/common/test_api.py
+5 -2
View File
@@ -158,7 +158,8 @@ class EndPoints(object):
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data), 7)
self.assertEqual(len(result_data['positive']), 7)
self.assertEqual(len(result_data['negative']), 7)
def test_diff_exp_indices(self):
endpoint = "diffexp/obs"
@@ -173,7 +174,8 @@ class EndPoints(object):
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data), 10)
self.assertEqual(len(result_data['positive']), 10)
self.assertEqual(len(result_data['negative']), 10)
def test_get_annotations_var_fbs(self):
endpoint = "annotations/var"
@@ -382,6 +384,7 @@ class EndPoints(object):
query_hash = hashlib.sha1(query.encode()).hexdigest()
url = f"{self.URL_BASE}{endpoint}?key={query_hash}"
result = self.session.post(url, headers=headers, data=query)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
backend/test/test_czi_hosted/unit/compute/test_diffexp_cxg.py
+34 -9
View File
@@ -4,7 +4,8 @@ import unittest
import numpy as np
from backend.czi_hosted.compute import diffexp_generic, diffexp_cxg
from backend.czi_hosted.compute import diffexp_cxg
from backend.common.compute import diffexp_generic
from backend.czi_hosted.compute.diffexp_cxg import diffexp_ttest
from backend.czi_hosted.converters.h5ad_data_file import H5ADDataFile
from backend.common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
@@ -40,21 +41,37 @@ class DiffExpTest(unittest.TestCase):
self.assertTrue(np.isclose(result[2], expect[2], 1e-6, 1e-4))
self.assertTrue(np.isclose(result[3], expect[3], 1e-6, 1e-4))
def check_1_10_2_10(self, results):
"""Checks the results for a specific set of rows selections"""
expects = [
positive_expects = [
[1712, -0.5525154, 0.0051788902660723345, 1.0],
[1575, 1.0317602, 0.007830310753043345, 1.0],
[693, 0.4703904, 0.008715846769131548, 1.0],
[916, 0.9567287, 0.009080596532247588, 1.0],
[77, 0.02665649, 0.010070392939027756, 1.0],
[782, -1.0981874, 0.010161745218916036, 1.0],
[913, 0.5683986, 0.010782030711612685, 1.0],
[910, 0.83164597, 0.014596411069229197, 1.0],
[1727, 0.4127781, 0.015168372104237176, 1.0],
[1443, -0.8241895, 0.015337080567465522, 1.0]
]
negative_expects = [
[956, 0.016060986, 0.0008649321884808977, 1.0],
[1124, 0.96602094, 0.0011717216548271284, 1.0],
[1809, 1.1110606, 0.0019304405196777848, 1.0],
[1712, -0.5525154, 0.0051788902660723345, 1.0],
[1754, 0.5201581, 0.005691734062127954, 1.0],
[948, 1.6390722, 0.006622111055981219, 1.0],
[1810, 0.78618884, 0.007055917428377063, 1.0],
[779, 1.5241305, 0.007202934422407284, 1.0],
[1575, 1.0317602, 0.007830310753043345, 1.0],
[576, 0.97873515, 0.008272092578813124, 1.0],
[538, 0.89114505, 0.01062259019889307, 1.0],
[436, 0.3119122, 0.01127515110543434, 1.0]
]
self.compare_diffexp_results(results, expects)
self.compare_diffexp_results(results['positive'], positive_expects)
self.compare_diffexp_results(results['negative'], negative_expects)
def get_X_col(self, adaptor, cols):
varmask = np.zeros(adaptor.get_shape()[1], dtype=bool)
@@ -80,6 +97,7 @@ class DiffExpTest(unittest.TestCase):
self.check_1_10_2_10(results)
# run it directly
results = diffexp_ttest(adaptor, maskA, maskB, 10)
self.check_1_10_2_10(results)
@@ -128,15 +146,22 @@ class DiffExpTest(unittest.TestCase):
diffexp_results_sparse = diffexp_cxg.diffexp_ttest(adaptor_sparse, maskA, maskB, 10)
diffexp_results_dense = diffexp_cxg.diffexp_ttest(adaptor_dense, maskA, maskB, 10)
self.compare_diffexp_results(diffexp_results_anndata, diffexp_results_sparse)
self.compare_diffexp_results(diffexp_results_anndata, diffexp_results_dense)
self.compare_diffexp_results(diffexp_results_anndata['positive'], diffexp_results_sparse['positive'])
self.compare_diffexp_results(diffexp_results_anndata['negative'], diffexp_results_sparse['negative'])
self.compare_diffexp_results(diffexp_results_anndata['positive'], diffexp_results_dense['positive'])
self.compare_diffexp_results(diffexp_results_anndata['negative'], diffexp_results_dense['negative'])
topcols_pos = np.array([x[0] for x in diffexp_results_anndata['positive']])
topcols_neg = np.array([x[0] for x in diffexp_results_anndata['negative']])
topcols = np.concatenate((topcols_pos, topcols_neg))
topcols = np.array([x[0] for x in diffexp_results_anndata])
cols_anndata = self.get_X_col(adaptor_anndata, topcols)
cols_sparse = self.get_X_col(adaptor_sparse, topcols)
cols_dense = self.get_X_col(adaptor_dense, topcols)
assert cols_anndata.shape[0] == adaptor_sparse.get_shape()[0]
assert cols_anndata.shape[1] == len(diffexp_results_anndata)
assert cols_anndata.shape[1] == len(diffexp_results_anndata['positive']) + len(diffexp_results_anndata['negative'])
def convert(mat, cols):
return decode_matrix_fbs(encode_matrix_fbs(mat, col_idx=cols)).to_numpy()
backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor.py
+4 -2
View File
@@ -152,9 +152,11 @@ class AdaptorTest(unittest.TestCase):
f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
self.assertEqual(len(result), 10)
self.assertEqual(len(result['positive']), 10)
self.assertEqual(len(result['negative']), 10)
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
self.assertEqual(len(result), 20)
self.assertEqual(len(result['positive']), 20)
self.assertEqual(len(result['negative']), 20)
def test_data_frame(self):
f1 = {"var": {"index": [[0, 10]]}}
backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor_data_load.py
+7 -2
View File
@@ -30,10 +30,15 @@ class DataLoadAdaptorTest(unittest.TestCase):
def test_diffexp_topN(self):
f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
self.assertEqual(len(result), 10)
self.assertEqual(len(result['positive']), 10)
self.assertEqual(len(result['negative']), 10)
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
self.assertEqual(len(result), 20)
self.assertEqual(len(result['positive']), 20)
self.assertEqual(len(result['negative']), 20)
class DataLocatorAdaptorTest(unittest.TestCase):
backend/test/test_server/performance/run_diffexp.py
+1 -1
View File
@@ -4,7 +4,7 @@ import random
import time
import numpy as np
import backend.server.compute.diffexp_generic as diffexp_generic
import backend.common.compute.diffexp_generic as diffexp_generic
from backend.server.common.config.app_config import AppConfig
from backend.server.data_common.matrix_loader import MatrixDataLoader
backend/test/test_server/unit/common/test_api.py
+4 -2
View File
@@ -414,7 +414,8 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data), 7)
self.assertEqual(len(result_data['positive']), 7)
self.assertEqual(len(result_data['negative']), 7)
def test_diff_exp_indices(self):
endpoint = "diffexp/obs"
@@ -429,7 +430,8 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data), 10)
self.assertEqual(len(result_data['positive']), 10)
self.assertEqual(len(result_data['negative']), 10)
def test_get_summaryvar(self):
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
backend/test/test_server/unit/compute/test_diffexp_cxg.py → backend/test/test_server/unit/compute/test_diffexp_h5ad.py
+47 -5
View File
@@ -2,13 +2,14 @@ import unittest
import numpy as np
from backend.common.compute import diffexp_generic
from backend.server.data_common.matrix_loader import MatrixDataLoader
from backend.test.test_server.unit import app_config
from backend.test import PROJECT_ROOT
class DiffExpTest(unittest.TestCase):
"""Tests the diffexp returns the expected results for one test case, using different
"""Tests the diffexp returns the expected results for one test case, using the h5ad
adaptor types and different algorithms."""
def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}):
@@ -35,19 +36,34 @@ class DiffExpTest(unittest.TestCase):
def check_1_10_2_10(self, results):
"""Checks the results for a specific set of rows selections"""
expects = [
positive_expects = [
[1712, -0.5525154, 0.0051788902660723345, 1.0],
[1575, 1.0317602, 0.007830310753043345, 1.0],
[693, 0.4703904, 0.008715846769131548, 1.0],
[916, 0.9567287, 0.009080596532247588, 1.0],
[77, 0.02665649, 0.010070392939027756, 1.0],
[782, -1.0981874, 0.010161745218916036, 1.0],
[913, 0.5683986, 0.010782030711612685, 1.0],
[910, 0.83164597, 0.014596411069229197, 1.0],
[1727, 0.4127781, 0.015168372104237176, 1.0],
[1443, -0.8241895, 0.015337080567465522, 1.0]
]
negative_expects = [
[956, 0.016060986, 0.0008649321884808977, 1.0],
[1124, 0.96602094, 0.0011717216548271284, 1.0],
[1809, 1.1110606, 0.0019304405196777848, 1.0],
[1712, -0.5525154, 0.0051788902660723345, 1.0],
[1754, 0.5201581, 0.005691734062127954, 1.0],
[948, 1.6390722, 0.006622111055981219, 1.0],
[1810, 0.78618884, 0.007055917428377063, 1.0],
[779, 1.5241305, 0.007202934422407284, 1.0],
[1575, 1.0317602, 0.007830310753043345, 1.0],
[576, 0.97873515, 0.008272092578813124, 1.0],
[538, 0.89114505, 0.01062259019889307, 1.0],
[436, 0.3119122, 0.01127515110543434, 1.0]
]
self.compare_diffexp_results(results, expects)
self.compare_diffexp_results(results["positive"], positive_expects)
self.compare_diffexp_results(results["negative"], negative_expects)
def get_X_col(self, adaptor, cols):
varmask = np.zeros(adaptor.get_shape()[1], dtype=bool)
@@ -61,3 +77,29 @@ class DiffExpTest(unittest.TestCase):
maskB = self.get_mask(adaptor, 2, 10)
results = adaptor.compute_diffexp_ttest(maskA, maskB, 10)
self.check_1_10_2_10(results)
def test_h5ad_default(self):
"""Test a h5ad adaptor with its default diffexp algorithm (diffexp_cxg)"""
adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
maskA = self.get_mask(adaptor, 1, 10)
maskB = self.get_mask(adaptor, 2, 10)
# run it through the adaptor
results = adaptor.compute_diffexp_ttest(maskA, maskB, 10)
self.check_1_10_2_10(results)
# run it directly
results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB, 10)
self.check_1_10_2_10(results)
def test_h5ad_generic(self):
"""Test a h5ad adaptor with the generic adaptor"""
adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
maskA = self.get_mask(adaptor, 1, 10)
maskB = self.get_mask(adaptor, 2, 10)
# run it directly
results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB, 10)
self.check_1_10_2_10(results)
backend/test/test_server/unit/data_anndata/test_anndata_adaptor.py
+5 -2
View File
@@ -153,9 +153,12 @@ class AdaptorTest(unittest.TestCase):
f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
self.assertEqual(len(result), 10)
self.assertEqual(len(result['positive']), 10)
self.assertEqual(len(result['negative']), 10)
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
self.assertEqual(len(result), 20)
self.assertEqual(len(result['positive']), 20)
self.assertEqual(len(result['negative']), 20)
def test_data_frame(self):
f1 = {"var": {"index": [[0, 10]]}}
backend/test/test_server/unit/data_anndata/test_anndata_adaptor_data_load.py
+4 -2
View File
@@ -31,9 +31,11 @@ class DataLoadAdaptorTest(unittest.TestCase):
f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
self.assertEqual(len(result), 10)
self.assertEqual(len(result['positive']), 10)
self.assertEqual(len(result['negative']), 10)
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
self.assertEqual(len(result), 20)
self.assertEqual(len(result['positive']), 20)
self.assertEqual(len(result['negative']), 20)
class DataLocatorAdaptorTest(unittest.TestCase):