fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-10-01 17:28:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

binary wire format with flatbuffers (#509)

Browse Source 
* first flatbuffer schema

* do not lint auto-generated files

* add flatbuffers package

* add flatbuffer module

* wire up /data/X/T route

* use flatbuffers for matrix data fetc

* clarity and comments

* add flatbuffer layout route

* clean up obsolete code

* fix tests

* move flake8 config to setup.cfg

* add comments

* lint

* rework layout routes for fbs

* add more type support to fbs

* lint

* add flatbuffer support for annotations

* function name improvements

* fix botched merge with master

* remove unused import

* route cleanup for flatbuffers

* rename function for clarity

* add missing globals to Jest tests

* fix client JS tests

* fix routes for Python tests

* comments for clarity

* non-finite floating point hardening

* more non-finite number handling

* lint

* fix tests for summarizeAnnotations

* harden diffexp calculation against FP errors

* cleanup unused code

* lint

* add encoding tests for flatbuffers

* application type specified as strings

* fix spelling error

* improve variable names

* add note about documentation gap

* rename FBS DataFrame to Matrix
This commit is contained in:
Bruce Martin
2019-01-09 14:26:05 -08:00
committed by GitHub
parent 42e25a1a1f
commit b90447c387
41 changed files with 2317 additions and 266 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/test/test_api.py
+94 -1
View File
@@ -5,6 +5,8 @@ import time
import requests
import decode_fbs
LOCAL_URL = "http://127.0.0.1:5005/"
VERSION = "v0.2"
URL_BASE = f"{LOCAL_URL}api/{VERSION}/"
@@ -40,6 +42,7 @@ class EndPoints(unittest.TestCase):
url = f"{URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["schema"]["dataframe"]["nObs"], 2638)
self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 5)
@@ -49,6 +52,7 @@ class EndPoints(unittest.TestCase):
url = f"{URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k")
self.assertEqual(len(result_data["config"]["features"]), 4)
@@ -58,10 +62,26 @@ class EndPoints(unittest.TestCase):
url = f"{URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["layout"]["ndims"], 2)
self.assertEqual(len(result_data["layout"]["coordinates"]), 2638)
def test_get_layout_fbs(self):
endpoint = "layout/obs"
url = f"{URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 2)
self.assertIsNotNone(df['columns'])
self.assertIsNone(df['col_idx'])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
# def test_put_layout(self):
# endpoint = "layout/obs"
# url = f"{URL_BASE}{endpoint}"
@@ -93,6 +113,7 @@ class EndPoints(unittest.TestCase):
url = f"{URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["names"], ["name", "n_genes", "percent_mito", "n_counts", "louvain"])
self.assertEqual(len(result_data["data"]), 2638)
@@ -103,11 +124,28 @@ class EndPoints(unittest.TestCase):
query = "annotation-name=n_genes&annotation-name=percent_mito"
url = f"{URL_BASE}{endpoint}?{query}"
result = self.session.get(url)
self.assertEqual(result.headers["Content-Type"], "application/json")
self.assertEqual(result.status_code, HTTPStatus.OK)
result_data = result.json()
self.assertEqual(result_data["names"], ["n_genes", "percent_mito"])
self.assertEqual(len(result_data["data"][0]), 3)
def test_get_annotations_obs_fbs(self):
endpoint = "annotations/obs"
url = f"{URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 5)
self.assertIsNotNone(df['columns'])
self.assertIsNotNone(df['col_idx'])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
self.assertListEqual(df['col_idx'], ['name', 'n_genes', 'percent_mito', 'n_counts', 'louvain'])
def test_get_annotations_obs_error(self):
endpoint = "annotations/obs"
query = "annotation-name=notakey"
@@ -131,6 +169,7 @@ class EndPoints(unittest.TestCase):
}
result = self.session.put(url, json=obs_filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["names"], ["name", "n_genes", "percent_mito", "n_counts", "louvain"])
self.assertEqual(len(result_data["data"]), 15)
@@ -152,6 +191,7 @@ class EndPoints(unittest.TestCase):
}
result = self.session.put(url, json=obs_filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["names"], ["n_genes", "percent_mito"])
self.assertEqual(len(result_data["data"][0]), 3)
@@ -168,6 +208,7 @@ class EndPoints(unittest.TestCase):
}
result = self.session.post(url, json=params)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data), 7)
@@ -182,6 +223,7 @@ class EndPoints(unittest.TestCase):
}
result = self.session.post(url, json=params)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data), 10)
@@ -190,6 +232,7 @@ class EndPoints(unittest.TestCase):
url = f"{URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["names"], ["name", "n_cells"])
self.assertEqual(len(result_data["data"]), 1838)
@@ -201,10 +244,27 @@ class EndPoints(unittest.TestCase):
url = f"{URL_BASE}{endpoint}?{query}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["names"], ["n_cells"])
self.assertEqual(len(result_data["data"][0]), 2)
def test_get_annotations_var_fbs(self):
endpoint = "annotations/var"
url = f"{URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 1838)
self.assertEqual(df['n_cols'], 2)
self.assertIsNotNone(df['columns'])
self.assertIsNotNone(df['col_idx'])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
self.assertListEqual(df['col_idx'], ['name', 'n_cells'])
def test_get_annotations_var_error(self):
endpoint = "annotations/var"
query = "annotation-name=notakey"
@@ -218,6 +278,7 @@ class EndPoints(unittest.TestCase):
var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}]}}}
result = self.session.put(url, json=var_filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["names"], ["name", "n_cells"])
self.assertEqual(len(result_data["data"]), 2)
@@ -229,6 +290,7 @@ class EndPoints(unittest.TestCase):
var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}]}}}
result = self.session.put(url, json=var_filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["names"], ["n_cells"])
self.assertEqual(len(result_data["data"][0]), 2)
@@ -241,6 +303,7 @@ class EndPoints(unittest.TestCase):
url = f"{URL_BASE}{endpoint}?{query}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data["obs"]), 2638)
@@ -262,6 +325,7 @@ class EndPoints(unittest.TestCase):
url = f"{URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
def test_data_filter(self):
for axis in ["obs", "var"]:
@@ -270,10 +334,11 @@ class EndPoints(unittest.TestCase):
url = f"{URL_BASE}{endpoint}?{query}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data["obs"]), 38)
def test_data_put(self):
def test_data_json_put(self):
for axis in ["obs", "var"]:
endpoint = f"data/{axis}"
url = f"{URL_BASE}{endpoint}"
@@ -291,9 +356,33 @@ class EndPoints(unittest.TestCase):
}
result = self.session.put(url, headers=header, json=obs_filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data["obs"]), 15)
def test_data_put_fbs(self):
endpoint = f"data/var"
url = f"{URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
filter = {
"filter": {
"var": {
"index": [0, 1, 4]
}
}
}
result = self.session.put(url, headers=header, json=filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 3)
self.assertIsNotNone(df['columns'])
self.assertIsNotNone(df['col_idx'])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
self.assertListEqual(df['col_idx'].tolist(), [0, 1, 4])
def test_data_put_single_var(self):
for axis in ["obs", "var"]:
endpoint = f"data/{axis}"
@@ -302,6 +391,7 @@ class EndPoints(unittest.TestCase):
var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}}}
result = self.session.put(url, headers=header, json=var_filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
if axis == "obs":
self.assertEqual(len(result_data["obs"][0]), 2)
@@ -338,6 +428,7 @@ class EndPoints(unittest.TestCase):
}
result = self.session.put(url, json=f1)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data1 = result.json()
f2 = {
"filter": {
@@ -353,6 +444,7 @@ class EndPoints(unittest.TestCase):
}
result = self.session.put(url, json=f2)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data2 = result.json()
self.assertNotEqual(result_data1, result_data2)
@@ -377,6 +469,7 @@ class EndPoints(unittest.TestCase):
}
result = self.session.put(url, json=f2)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data2 = result.json()
self.assertNotEqual(result_data1, result_data2)