fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-10-02 12:28:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Move jsonification to engine level (#511)

Browse Source
This commit is contained in:
Charlotte Weaver
2018-12-18 21:14:24 -08:00
committed by GitHub
parent 6af7708d62
commit f0f7200f0b
10 changed files with 657 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/test/test_datasets/nan.h5ad
BIN
View File
Binary file not shown.
server/test/test_nan_rest.py
+96
View File
@@ -0,0 +1,96 @@
from http import HTTPStatus
from subprocess import Popen
import unittest
import time
import requests
LOCAL_URL = "http://127.0.0.1:5005/"
VERSION = "v0.2"
URL_BASE = f"{LOCAL_URL}api/{VERSION}/"
BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
class WithNaNs(unittest.TestCase):
"""Test Case for endpoints"""
@classmethod
def setUpClass(cls):
cls.ps = Popen(
["cellxgene", "launch", "server/test/test_datasets/nan.h5ad", "--debug"]
)
session = requests.Session()
for i in range(90):
try:
session.get(f"{URL_BASE}schema")
except requests.exceptions.ConnectionError:
time.sleep(1)
@classmethod
def tearDownClass(cls):
try:
cls.ps.terminate()
except ProcessLookupError:
pass
def setUp(self):
self.session = requests.Session()
def test_initialize(self):
endpoint = "schema"
url = f"{URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
def test_errors(self):
endpoints = ["annotations/obs", "annotations/var", "data/obs", "data/var"]
for endpoint in endpoints:
url = f"{URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.INTERNAL_SERVER_ERROR)
class WithoutNaNs(unittest.TestCase):
"""Test Case for endpoints"""
@classmethod
def setUpClass(cls):
cls.ps = Popen(
[
"cellxgene",
"launch",
"server/test/test_datasets/nan.h5ad",
"--nan-to-num",
"--debug",
]
)
session = requests.Session()
for i in range(90):
try:
session.get(f"{URL_BASE}schema")
except requests.exceptions.ConnectionError:
time.sleep(1)
@classmethod
def tearDownClass(cls):
try:
cls.ps.terminate()
except ProcessLookupError:
pass
def setUp(self):
self.session = requests.Session()
def test_initialize(self):
endpoint = "schema"
url = f"{URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
def test_errors(self):
endpoints = ["annotations/obs", "annotations/var", "data/obs", "data/var"]
for endpoint in endpoints:
url = f"{URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
server/test/test_nan_scanpy_engine.py
+85
View File
@@ -0,0 +1,85 @@
import json
import pytest
import unittest
import warnings
from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
from server.app.util.errors import JSONEncodingValueError
class NaNTest(unittest.TestCase):
def setUp(self):
self.args = {
"layout": "umap",
"diffexp": "ttest",
"max_category_items": 100,
"obs_names": None,
"var_names": None,
"diffexp_lfc_cutoff": 0.01,
"nan_to_num": False,
}
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UserWarning)
self.data = ScanpyEngine("server/test/test_datasets/nan.h5ad", self.args)
self.data._create_schema()
self.args_nan = dict(self.args)
self.args_nan["nan_to_num"] = True
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UserWarning)
self.data_nan = ScanpyEngine(
"server/test/test_datasets/nan.h5ad", self.args_nan
)
self.data_nan._create_schema()
def test_load(self):
with self.assertWarns(UserWarning):
ScanpyEngine("server/test/test_datasets/nan.h5ad", self.args_nan)
def test_init(self):
self.assertEqual(self.data.cell_count, 100)
self.assertEqual(self.data.gene_count, 100)
epsilon = 0.000_005
self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
self.assertEqual(self.data_nan.cell_count, 100)
self.assertEqual(self.data_nan.gene_count, 100)
epsilon = 0.000_005
self.assertTrue(self.data_nan.data.X[0, 0] - -0.171_469_51 < epsilon)
def test_dataframe(self):
data_frame_obs = json.loads(self.data_nan.data_frame(None, "obs"))
self.assertEqual(len(data_frame_obs["var"]), 100)
self.assertEqual(len(data_frame_obs["obs"]), 100)
data_frame_var = json.loads(self.data_nan.data_frame(None, "var"))
self.assertEqual(len(data_frame_var["var"]), 100)
self.assertEqual(len(data_frame_var["obs"]), 100)
with pytest.raises(JSONEncodingValueError):
data_frame_obs = json.loads(self.data.data_frame(None, "obs"))
with pytest.raises(JSONEncodingValueError):
data_frame_var = json.loads(self.data.data_frame(None, "var"))
def test_dataframe_nan_to_0(self):
data_frame_obs = json.loads(self.data_nan.data_frame(None, "obs"))
self.assertEqual(data_frame_obs["obs"][1][3], 0.0)
data_frame_var = json.loads(self.data_nan.data_frame(None, "var"))
self.assertEqual(data_frame_var["var"][1][5], 0.0)
def test_annotation_nan_to_0(self):
annotations_obs = json.loads(self.data_nan.annotation(None, "obs"))
self.assertEqual(annotations_obs["data"][0][3], 0.0)
annotations_var = json.loads(self.data_nan.annotation(None, "var"))
self.assertEqual(annotations_var["data"][0][3], 0.0)
def test_annotation(self):
annotations = json.loads(self.data_nan.annotation(None, "obs"))
self.assertEqual(
annotations["names"],
["name", "n_genes", "percent_mito", "n_counts", "louvain"],
)
annotations = json.loads(self.data_nan.annotation(None, "var"))
self.assertEqual(annotations["names"], ["name", "n_cells", "var_with_nans"])
self.assertEqual(len(annotations["data"]), 100)
with pytest.raises(JSONEncodingValueError):
annotations = json.loads(self.data.annotation(None, "obs"))
with pytest.raises(JSONEncodingValueError):
annotations = json.loads(self.data.annotation(None, "var"))
server/test/test_scanpy_engine.py
+71 -32
View File
@@ -44,22 +44,39 @@ class UtilTest(unittest.TestCase):
self.data._validate_data_types()
def test_filter_idx(self):
filter_ = {"filter": {"var": {"index": [1, 99, [200, 300]]}, "obs": {"index": [1, 99, [1000, 2000]]}}}
filter_ = {
"filter": {
"var": {"index": [1, 99, [200, 300]]},
"obs": {"index": [1, 99, [1000, 2000]]},
}
}
data = self.data.filter_dataframe(filter_["filter"])
self.assertEqual(data.shape, (1002, 102))
def test_filter_annotation(self):
filter_ = {
"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["NK cells", "CD8 T cells"]}]}}
"filter": {
"obs": {
"annotation_value": [
{"name": "louvain", "values": ["NK cells", "CD8 T cells"]}
]
}
}
}
data = self.data.filter_dataframe(filter_["filter"])
self.assertEqual(data.shape, (470, 1838))
filter_ = {"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}}
filter_ = {
"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}
}
data = self.data.filter_dataframe(filter_["filter"])
self.assertEqual(data.shape, (497, 1838))
def test_filter_annotation_no_uns(self):
filter_ = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}}}
filter_ = {
"filter": {
"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}
}
}
data = self.data.filter_dataframe(filter_["filter"])
self.assertEqual(data.shape[1], 1)
@@ -90,36 +107,45 @@ class UtilTest(unittest.TestCase):
def test_schema_produces_error(self):
self.data.data.obs["time"] = Series(
list([time.time() for i in range(self.data.cell_count)]), dtype="datetime64[ns]"
list([time.time() for i in range(self.data.cell_count)]),
dtype="datetime64[ns]",
)
with pytest.raises(TypeError):
self.data._create_schema()
def test_config(self):
self.assertEqual(self.data.features["layout"]["obs"], {"available": True, "interactiveLimit": 50000})
self.assertEqual(
self.data.features["layout"]["obs"],
{"available": True, "interactiveLimit": 50000},
)
def test_layout(self):
layout = self.data.layout(None)
self.assertEqual(layout["ndims"], 2)
self.assertEqual(len(layout["coordinates"]), 2638)
self.assertEqual(layout["coordinates"][0][0], 0)
for idx, val in enumerate(layout["coordinates"]):
layout = json.loads(self.data.layout(None))
self.assertEqual(layout["layout"]["ndims"], 2)
self.assertEqual(len(layout["layout"]["coordinates"]), 2638)
self.assertEqual(layout["layout"]["coordinates"][0][0], 0)
for idx, val in enumerate(layout["layout"]["coordinates"]):
self.assertLessEqual(val[1], 1)
self.assertLessEqual(val[2], 1)
def test_annotations(self):
annotations = self.data.annotation(None, "obs")
self.assertEqual(annotations["names"], ["name", "n_genes", "percent_mito", "n_counts", "louvain"])
annotations = json.loads(self.data.annotation(None, "obs"))
self.assertEqual(
annotations["names"],
["name", "n_genes", "percent_mito", "n_counts", "louvain"],
)
self.assertEqual(len(annotations["data"]), 2638)
annotations = self.data.annotation(None, "var")
annotations = json.loads(self.data.annotation(None, "var"))
self.assertEqual(annotations["names"], ["name", "n_cells"])
self.assertEqual(len(annotations["data"]), 1838)
def test_annotation_fields(self):
annotations = self.data.annotation(None, "obs", ["n_genes", "n_counts"])
annotations = json.loads(
self.data.annotation(None, "obs", ["n_genes", "n_counts"])
)
self.assertEqual(annotations["names"], ["n_genes", "n_counts"])
self.assertEqual(len(annotations["data"]), 2638)
annotations = self.data.annotation(None, "var", ["name"])
annotations = json.loads(self.data.annotation(None, "var", ["name"]))
self.assertEqual(annotations["names"], ["name"])
self.assertEqual(len(annotations["data"]), 1838)
@@ -127,45 +153,54 @@ class UtilTest(unittest.TestCase):
filter_ = {
"filter": {
"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]},
"var": {"annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}]},
"var": {
"annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}]
},
}
}
annotations = self.data.annotation(filter_["filter"], "obs")
self.assertEqual(annotations["names"], ["name", "n_genes", "percent_mito", "n_counts", "louvain"])
annotations = json.loads(self.data.annotation(filter_["filter"], "obs"))
self.assertEqual(
annotations["names"],
["name", "n_genes", "percent_mito", "n_counts", "louvain"],
)
self.assertEqual(len(annotations["data"]), 497)
annotations = self.data.annotation(filter_["filter"], "var")
annotations = json.loads(self.data.annotation(filter_["filter"], "var"))
self.assertEqual(annotations["names"], ["name", "n_cells"])
self.assertEqual(len(annotations["data"]), 2)
def test_filtered_layout(self):
filter_ = {"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}}
layout = self.data.layout(filter_["filter"])
self.assertEqual(len(layout["coordinates"]), 497)
filter_ = {
"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}
}
layout = json.loads(self.data.layout(filter_["filter"]))
self.assertEqual(len(layout["layout"]["coordinates"]), 497)
def test_diffexp_topN(self):
f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
result = self.data.diffexp_topN(f1["filter"], f2["filter"])
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
self.assertEqual(len(result), 10)
result = self.data.diffexp_topN(f1["filter"], f2["filter"], 20)
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
self.assertEqual(len(result), 20)
def test_data_frame(self):
data_frame_obs = self.data.data_frame(None, "obs")
data_frame_obs = json.loads(self.data.data_frame(None, "obs"))
self.assertEqual(len(data_frame_obs["var"]), 1838)
self.assertEqual(len(data_frame_obs["obs"]), 2638)
data_frame_var = self.data.data_frame(None, "var")
data_frame_var = json.loads(self.data.data_frame(None, "var"))
self.assertEqual(len(data_frame_var["var"]), 1838)
self.assertEqual(len(data_frame_var["obs"]), 2638)
def test_filtered_data_frame(self):
filter_ = {"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}}
data_frame_obs = self.data.data_frame(filter_["filter"], "obs")
filter_ = {
"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}
}
data_frame_obs = json.loads(self.data.data_frame(filter_["filter"], "obs"))
self.assertEqual(len(data_frame_obs["var"]), 1838)
self.assertEqual(len(data_frame_obs["obs"]), 497)
self.assertIsInstance(data_frame_obs["obs"][0], (list, tuple))
self.assertEqual(type(data_frame_obs["var"][0]), int)
data_frame_var = self.data.data_frame(filter_["filter"], "var")
data_frame_var = json.loads(self.data.data_frame(filter_["filter"], "var"))
self.assertEqual(len(data_frame_var["var"]), 1838)
self.assertEqual(len(data_frame_var["obs"]), 497)
self.assertIsInstance(data_frame_var["var"][0], (list, tuple))
@@ -173,8 +208,12 @@ class UtilTest(unittest.TestCase):
def test_data_single_gene(self):
for axis in ["obs", "var"]:
filter_ = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}}}
data_frame_var = self.data.data_frame(filter_["filter"], axis)
filter_ = {
"filter": {
"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}
}
}
data_frame_var = json.loads(self.data.data_frame(filter_["filter"], axis))
if axis == "obs":
self.assertEqual(type(data_frame_var["var"][0]), int)
self.assertIsInstance(data_frame_var["obs"][0], (list, tuple))