fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-28 20:18:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

do not hard-wire column names in annotations (#785)

Browse Source 
* enforce column name uniqueness for obs and var

* parameterize the column name containing obs and var user-readable names

* use the new annotation index value from schema

* update f/e unit tests

* PR review suggestions

* lint
This commit is contained in:
Bruce Martin
2019-05-24 21:00:54 -07:00
committed by GitHub
parent a8c2e408d1
commit 3dc45d6330
16 changed files with 246 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/test/schema.json
+48 -42
View File
@@ -5,48 +5,54 @@
"type": "float32"
},
"annotations": {
"obs": [
{
"name": "name",
"type": "string"
},
{
"name": "n_genes",
"type": "int32"
},
{
"name": "percent_mito",
"type": "float32"
},
{
"name": "n_counts",
"type": "float32"
},
{
"name": "louvain",
"type": "categorical",
"categories": [
"CD4 T cells",
"CD14+ Monocytes",
"B cells",
"CD8 T cells",
"NK cells",
"FCGR3A+ Monocytes",
"Dendritic cells",
"Megakaryocytes"
]
}
],
"var": [
{
"name": "name",
"type": "string"
},
{
"name": "n_cells",
"type": "int32"
}
]
"obs": {
"index": "name_0",
"columns": [
{
"name": "name_0",
"type": "string"
},
{
"name": "n_genes",
"type": "int32"
},
{
"name": "percent_mito",
"type": "float32"
},
{
"name": "n_counts",
"type": "float32"
},
{
"name": "louvain",
"type": "categorical",
"categories": [
"CD4 T cells",
"CD14+ Monocytes",
"B cells",
"CD8 T cells",
"NK cells",
"FCGR3A+ Monocytes",
"Dendritic cells",
"Megakaryocytes"
]
}
]
},
"var": {
"index": "name_0",
"columns": [
{
"name": "name_0",
"type": "string"
},
{
"name": "n_cells",
"type": "int32"
}
]
}
},
"layout": {
"obs": [
server/test/test_api.py
+10 -5
View File
@@ -23,7 +23,8 @@ class EndPoints(unittest.TestCase):
session = requests.Session()
for i in range(90):
try:
session.get(f"{URL_BASE}schema")
result = session.get(f"{URL_BASE}schema")
cls.schema = result.json()
except requests.exceptions.ConnectionError:
time.sleep(1)
@@ -45,7 +46,8 @@ class EndPoints(unittest.TestCase):
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["schema"]["dataframe"]["nObs"], 2638)
self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 5)
self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 2)
self.assertEqual(len(result_data["schema"]["annotations"]["obs"]["columns"]), 5)
def test_config(self):
endpoint = "config"
@@ -95,7 +97,8 @@ class EndPoints(unittest.TestCase):
self.assertIsNotNone(df['col_idx'])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
self.assertListEqual(df['col_idx'], ['name', 'n_genes', 'percent_mito', 'n_counts', 'louvain'])
obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"]
self.assertListEqual(df['col_idx'], [obs_index_col_name, 'n_genes', 'percent_mito', 'n_counts', 'louvain'])
def test_get_annotations_obs_keys_fbs(self):
endpoint = "annotations/obs"
@@ -165,7 +168,8 @@ class EndPoints(unittest.TestCase):
self.assertIsNotNone(df['col_idx'])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
self.assertListEqual(df['col_idx'], ['name', 'n_cells'])
var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
self.assertListEqual(df['col_idx'], [var_index_col_name, 'n_cells'])
def test_get_annotations_var_keys_fbs(self):
endpoint = "annotations/var"
@@ -247,7 +251,8 @@ class EndPoints(unittest.TestCase):
endpoint = f"data/var"
url = f"{URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}}}
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
var_filter = {"filter": {"var": {"annotation_value": [{"name": index_col_name, "values": ["RER1"]}]}}}
result = self.session.put(url, headers=header, json=var_filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
server/test/test_nan_scanpy_engine.py
+4 -2
View File
@@ -58,14 +58,16 @@ class NaNTest(unittest.TestCase):
def test_annotation(self):
annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("obs"))
obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
self.assertEqual(
annotations["col_idx"],
["name", "n_genes", "percent_mito", "n_counts", "louvain"]
[obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"]
)
self.assertEqual(annotations["n_rows"], 100)
self.assertTrue(math.isnan(annotations["columns"][2][0]))
annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("var"))
self.assertEqual(annotations["col_idx"], ["name", "n_cells", "var_with_nans"])
var_index_col_name = self.data.schema["annotations"]["var"]["index"]
self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells", "var_with_nans"])
self.assertEqual(annotations["n_rows"], 100)
self.assertTrue(math.isnan(annotations["columns"][2][0]))
server/test/test_scanpy_engine.py
+17 -9
View File
@@ -31,9 +31,11 @@ class EngineTest(unittest.TestCase):
self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
def test_mandatory_annotations(self):
self.assertIn("name", self.data.data.obs)
obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
self.assertIn(obs_index_col_name, self.data.data.obs)
self.assertEqual(list(self.data.data.obs.index), list(range(2638)))
self.assertIn("name", self.data.data.var)
var_index_col_name = self.data.schema["annotations"]["var"]["index"]
self.assertIn(var_index_col_name, self.data.data.var)
self.assertEqual(list(self.data.data.var.index), list(range(1838)))
@pytest.mark.filterwarnings("ignore:Scanpy data matrix")
@@ -70,12 +72,14 @@ class EngineTest(unittest.TestCase):
self.assertEqual(data["n_cols"], 91)
def test_obs_and_var_names(self):
self.assertEqual(np.sum(self.data.data.var["name"].isna()), 0)
self.assertEqual(np.sum(self.data.data.obs["name"].isna()), 0)
self.assertEqual(np.sum(self.data.data.var[self.data.schema["annotations"]["var"]["index"]].isna()), 0)
self.assertEqual(np.sum(self.data.data.obs[self.data.schema["annotations"]["obs"]["index"]].isna()), 0)
def test_schema(self):
with open(path.join(path.dirname(__file__), "schema.json")) as fh:
schema = json.load(fh)
print(schema)
print(self.data.schema)
self.assertEqual(self.data.schema, schema)
def test_schema_produces_error(self):
@@ -108,16 +112,18 @@ class EngineTest(unittest.TestCase):
annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations["n_rows"], 2638)
self.assertEqual(annotations["n_cols"], 5)
obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
self.assertEqual(
annotations["col_idx"],
["name", "n_genes", "percent_mito", "n_counts", "louvain"],
[obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
)
fbs = self.data.annotation_to_fbs_matrix("var")
annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations['n_rows'], 1838)
self.assertEqual(annotations['n_cols'], 2)
self.assertEqual(annotations["col_idx"], ["name", "n_cells"])
var_index_col_name = self.data.schema["annotations"]["var"]["index"]
self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells"])
def test_annotation_fields(self):
fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"])
@@ -125,7 +131,8 @@ class EngineTest(unittest.TestCase):
self.assertEqual(annotations["n_rows"], 2638)
self.assertEqual(annotations['n_cols'], 2)
fbs = self.data.annotation_to_fbs_matrix("var", ["name"])
var_index_col_name = self.data.schema["annotations"]["var"]["index"]
fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name])
annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations['n_rows'], 1838)
self.assertEqual(annotations['n_cols'], 1)
@@ -163,9 +170,10 @@ class EngineTest(unittest.TestCase):
self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
def test_data_named_gene(self):
var_index_col_name = self.data.schema["annotations"]["var"]["index"]
filter_ = {
"filter": {
"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}
"var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]}
}
}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
@@ -176,7 +184,7 @@ class EngineTest(unittest.TestCase):
filter_ = {
"filter": {
"var": {"annotation_value": [{"name": "name", "values": ["SPEN", "TYMP", "PRMT2"]}]}
"var": {"annotation_value": [{"name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"]}]}
}
}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")