mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-10-03 10:18:12 +08:00
binary wire format with flatbuffers (#509)
* first flatbuffer schema * do not lint auto-generated files * add flatbuffers package * add flatbuffer module * wire up /data/X/T route * use flatbuffers for matrix data fetc * clarity and comments * add flatbuffer layout route * clean up obsolete code * fix tests * move flake8 config to setup.cfg * add comments * lint * rework layout routes for fbs * add more type support to fbs * lint * add flatbuffer support for annotations * function name improvements * fix botched merge with master * remove unused import * route cleanup for flatbuffers * rename function for clarity * add missing globals to Jest tests * fix client JS tests * fix routes for Python tests * comments for clarity * non-finite floating point hardening * more non-finite number handling * lint * fix tests for summarizeAnnotations * harden diffexp calculation against FP errors * cleanup unused code * lint * add encoding tests for flatbuffers * application type specified as strings * fix spelling error * improve variable names * add note about documentation gap * rename FBS DataFrame to Matrix
This commit is contained in:
Bruce Martin
@@ -0,0 +1,69 @@
|
||||
|
||||
"""
|
||||
Code to decode, for testing purposes, the flatbuffer encoded blobs.
|
||||
This code will need to be updated if fbs/matrix.fbs changes.
|
||||
|
||||
For more information, see fbs/matrix.fbs and server/app/util/fbs/
|
||||
"""
|
||||
import json
|
||||
|
||||
import server.app.util.fbs.NetEncoding.TypedArray as TypedArray
|
||||
import server.app.util.fbs.NetEncoding.Matrix as Matrix
|
||||
import server.app.util.fbs.NetEncoding.Int32Array as Int32Array
|
||||
import server.app.util.fbs.NetEncoding.Uint32Array as Uint32Array
|
||||
import server.app.util.fbs.NetEncoding.Float32Array as Float32Array
|
||||
import server.app.util.fbs.NetEncoding.Float64Array as Float64Array
|
||||
import server.app.util.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
|
||||
|
||||
|
||||
def decode_typed_array(tarr):
|
||||
type_map = {
|
||||
TypedArray.TypedArray.Uint32Array: Uint32Array.Uint32Array,
|
||||
TypedArray.TypedArray.Int32Array: Int32Array.Int32Array,
|
||||
TypedArray.TypedArray.Float32Array: Float32Array.Float32Array,
|
||||
TypedArray.TypedArray.Float64Array: Float64Array.Float64Array,
|
||||
TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray
|
||||
}
|
||||
(u_type, u) = tarr
|
||||
if u_type == TypedArray.TypedArray.NONE:
|
||||
return None
|
||||
|
||||
TarType = type_map.get(u_type, None)
|
||||
assert(TarType is not None)
|
||||
|
||||
arr = TarType()
|
||||
arr.Init(u.Bytes, u.Pos)
|
||||
narr = arr.DataAsNumpy()
|
||||
if u_type == TypedArray.TypedArray.JSONEncodedArray:
|
||||
narr = json.loads(narr.tostring().decode('utf-8'))
|
||||
return narr
|
||||
|
||||
|
||||
def decode_matrix_FBS(buf):
|
||||
"""
|
||||
Given a FBS Matrix, return an decoded Python dict containing
|
||||
same info in native format.
|
||||
|
||||
NOTE / TODO: row_idx not currently implemented
|
||||
"""
|
||||
df = Matrix.Matrix.GetRootAsMatrix(buf, 0)
|
||||
n_rows = df.NRows()
|
||||
n_cols = df.NCols()
|
||||
|
||||
columns_length = df.ColumnsLength()
|
||||
|
||||
decoded_columns = []
|
||||
for col_idx in range(0, columns_length):
|
||||
col = df.Columns(col_idx)
|
||||
tarr = (col.UType(), col.U())
|
||||
decoded_columns.append(decode_typed_array(tarr))
|
||||
|
||||
cidx = decode_typed_array((df.ColIndexType(), df.ColIndex()))
|
||||
|
||||
return {
|
||||
"n_rows": n_rows,
|
||||
"n_cols": n_cols,
|
||||
"columns": decoded_columns,
|
||||
"col_idx": cidx,
|
||||
"row_idx": None
|
||||
}
|
||||
+94
-1
@@ -5,6 +5,8 @@ import time
|
||||
|
||||
import requests
|
||||
|
||||
import decode_fbs
|
||||
|
||||
LOCAL_URL = "http://127.0.0.1:5005/"
|
||||
VERSION = "v0.2"
|
||||
URL_BASE = f"{LOCAL_URL}api/{VERSION}/"
|
||||
@@ -40,6 +42,7 @@ class EndPoints(unittest.TestCase):
|
||||
url = f"{URL_BASE}{endpoint}"
|
||||
result = self.session.get(url)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
result_data = result.json()
|
||||
self.assertEqual(result_data["schema"]["dataframe"]["nObs"], 2638)
|
||||
self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 5)
|
||||
@@ -49,6 +52,7 @@ class EndPoints(unittest.TestCase):
|
||||
url = f"{URL_BASE}{endpoint}"
|
||||
result = self.session.get(url)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
result_data = result.json()
|
||||
self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k")
|
||||
self.assertEqual(len(result_data["config"]["features"]), 4)
|
||||
@@ -58,10 +62,26 @@ class EndPoints(unittest.TestCase):
|
||||
url = f"{URL_BASE}{endpoint}"
|
||||
result = self.session.get(url)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
result_data = result.json()
|
||||
self.assertEqual(result_data["layout"]["ndims"], 2)
|
||||
self.assertEqual(len(result_data["layout"]["coordinates"]), 2638)
|
||||
|
||||
def test_get_layout_fbs(self):
|
||||
endpoint = "layout/obs"
|
||||
url = f"{URL_BASE}{endpoint}"
|
||||
header = {"Accept": "application/octet-stream"}
|
||||
result = self.session.get(url, headers=header)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
|
||||
df = decode_fbs.decode_matrix_FBS(result.content)
|
||||
self.assertEqual(df['n_rows'], 2638)
|
||||
self.assertEqual(df['n_cols'], 2)
|
||||
self.assertIsNotNone(df['columns'])
|
||||
self.assertIsNone(df['col_idx'])
|
||||
self.assertIsNone(df['row_idx'])
|
||||
self.assertEqual(len(df['columns']), df['n_cols'])
|
||||
|
||||
# def test_put_layout(self):
|
||||
# endpoint = "layout/obs"
|
||||
# url = f"{URL_BASE}{endpoint}"
|
||||
@@ -93,6 +113,7 @@ class EndPoints(unittest.TestCase):
|
||||
url = f"{URL_BASE}{endpoint}"
|
||||
result = self.session.get(url)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
result_data = result.json()
|
||||
self.assertEqual(result_data["names"], ["name", "n_genes", "percent_mito", "n_counts", "louvain"])
|
||||
self.assertEqual(len(result_data["data"]), 2638)
|
||||
@@ -103,11 +124,28 @@ class EndPoints(unittest.TestCase):
|
||||
query = "annotation-name=n_genes&annotation-name=percent_mito"
|
||||
url = f"{URL_BASE}{endpoint}?{query}"
|
||||
result = self.session.get(url)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
result_data = result.json()
|
||||
self.assertEqual(result_data["names"], ["n_genes", "percent_mito"])
|
||||
self.assertEqual(len(result_data["data"][0]), 3)
|
||||
|
||||
def test_get_annotations_obs_fbs(self):
|
||||
endpoint = "annotations/obs"
|
||||
url = f"{URL_BASE}{endpoint}"
|
||||
header = {"Accept": "application/octet-stream"}
|
||||
result = self.session.get(url, headers=header)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
|
||||
df = decode_fbs.decode_matrix_FBS(result.content)
|
||||
self.assertEqual(df['n_rows'], 2638)
|
||||
self.assertEqual(df['n_cols'], 5)
|
||||
self.assertIsNotNone(df['columns'])
|
||||
self.assertIsNotNone(df['col_idx'])
|
||||
self.assertIsNone(df['row_idx'])
|
||||
self.assertEqual(len(df['columns']), df['n_cols'])
|
||||
self.assertListEqual(df['col_idx'], ['name', 'n_genes', 'percent_mito', 'n_counts', 'louvain'])
|
||||
|
||||
def test_get_annotations_obs_error(self):
|
||||
endpoint = "annotations/obs"
|
||||
query = "annotation-name=notakey"
|
||||
@@ -131,6 +169,7 @@ class EndPoints(unittest.TestCase):
|
||||
}
|
||||
result = self.session.put(url, json=obs_filter)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
result_data = result.json()
|
||||
self.assertEqual(result_data["names"], ["name", "n_genes", "percent_mito", "n_counts", "louvain"])
|
||||
self.assertEqual(len(result_data["data"]), 15)
|
||||
@@ -152,6 +191,7 @@ class EndPoints(unittest.TestCase):
|
||||
}
|
||||
result = self.session.put(url, json=obs_filter)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
result_data = result.json()
|
||||
self.assertEqual(result_data["names"], ["n_genes", "percent_mito"])
|
||||
self.assertEqual(len(result_data["data"][0]), 3)
|
||||
@@ -168,6 +208,7 @@ class EndPoints(unittest.TestCase):
|
||||
}
|
||||
result = self.session.post(url, json=params)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
result_data = result.json()
|
||||
self.assertEqual(len(result_data), 7)
|
||||
|
||||
@@ -182,6 +223,7 @@ class EndPoints(unittest.TestCase):
|
||||
}
|
||||
result = self.session.post(url, json=params)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
result_data = result.json()
|
||||
self.assertEqual(len(result_data), 10)
|
||||
|
||||
@@ -190,6 +232,7 @@ class EndPoints(unittest.TestCase):
|
||||
url = f"{URL_BASE}{endpoint}"
|
||||
result = self.session.get(url)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
result_data = result.json()
|
||||
self.assertEqual(result_data["names"], ["name", "n_cells"])
|
||||
self.assertEqual(len(result_data["data"]), 1838)
|
||||
@@ -201,10 +244,27 @@ class EndPoints(unittest.TestCase):
|
||||
url = f"{URL_BASE}{endpoint}?{query}"
|
||||
result = self.session.get(url)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
result_data = result.json()
|
||||
self.assertEqual(result_data["names"], ["n_cells"])
|
||||
self.assertEqual(len(result_data["data"][0]), 2)
|
||||
|
||||
def test_get_annotations_var_fbs(self):
|
||||
endpoint = "annotations/var"
|
||||
url = f"{URL_BASE}{endpoint}"
|
||||
header = {"Accept": "application/octet-stream"}
|
||||
result = self.session.get(url, headers=header)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
|
||||
df = decode_fbs.decode_matrix_FBS(result.content)
|
||||
self.assertEqual(df['n_rows'], 1838)
|
||||
self.assertEqual(df['n_cols'], 2)
|
||||
self.assertIsNotNone(df['columns'])
|
||||
self.assertIsNotNone(df['col_idx'])
|
||||
self.assertIsNone(df['row_idx'])
|
||||
self.assertEqual(len(df['columns']), df['n_cols'])
|
||||
self.assertListEqual(df['col_idx'], ['name', 'n_cells'])
|
||||
|
||||
def test_get_annotations_var_error(self):
|
||||
endpoint = "annotations/var"
|
||||
query = "annotation-name=notakey"
|
||||
@@ -218,6 +278,7 @@ class EndPoints(unittest.TestCase):
|
||||
var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}]}}}
|
||||
result = self.session.put(url, json=var_filter)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
result_data = result.json()
|
||||
self.assertEqual(result_data["names"], ["name", "n_cells"])
|
||||
self.assertEqual(len(result_data["data"]), 2)
|
||||
@@ -229,6 +290,7 @@ class EndPoints(unittest.TestCase):
|
||||
var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}]}}}
|
||||
result = self.session.put(url, json=var_filter)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
result_data = result.json()
|
||||
self.assertEqual(result_data["names"], ["n_cells"])
|
||||
self.assertEqual(len(result_data["data"][0]), 2)
|
||||
@@ -241,6 +303,7 @@ class EndPoints(unittest.TestCase):
|
||||
url = f"{URL_BASE}{endpoint}?{query}"
|
||||
result = self.session.get(url)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
result_data = result.json()
|
||||
self.assertEqual(len(result_data["obs"]), 2638)
|
||||
|
||||
@@ -262,6 +325,7 @@ class EndPoints(unittest.TestCase):
|
||||
url = f"{URL_BASE}{endpoint}"
|
||||
result = self.session.get(url)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
|
||||
def test_data_filter(self):
|
||||
for axis in ["obs", "var"]:
|
||||
@@ -270,10 +334,11 @@ class EndPoints(unittest.TestCase):
|
||||
url = f"{URL_BASE}{endpoint}?{query}"
|
||||
result = self.session.get(url)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
result_data = result.json()
|
||||
self.assertEqual(len(result_data["obs"]), 38)
|
||||
|
||||
def test_data_put(self):
|
||||
def test_data_json_put(self):
|
||||
for axis in ["obs", "var"]:
|
||||
endpoint = f"data/{axis}"
|
||||
url = f"{URL_BASE}{endpoint}"
|
||||
@@ -291,9 +356,33 @@ class EndPoints(unittest.TestCase):
|
||||
}
|
||||
result = self.session.put(url, headers=header, json=obs_filter)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
result_data = result.json()
|
||||
self.assertEqual(len(result_data["obs"]), 15)
|
||||
|
||||
def test_data_put_fbs(self):
|
||||
endpoint = f"data/var"
|
||||
url = f"{URL_BASE}{endpoint}"
|
||||
header = {"Accept": "application/octet-stream"}
|
||||
filter = {
|
||||
"filter": {
|
||||
"var": {
|
||||
"index": [0, 1, 4]
|
||||
}
|
||||
}
|
||||
}
|
||||
result = self.session.put(url, headers=header, json=filter)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
|
||||
df = decode_fbs.decode_matrix_FBS(result.content)
|
||||
self.assertEqual(df['n_rows'], 2638)
|
||||
self.assertEqual(df['n_cols'], 3)
|
||||
self.assertIsNotNone(df['columns'])
|
||||
self.assertIsNotNone(df['col_idx'])
|
||||
self.assertIsNone(df['row_idx'])
|
||||
self.assertEqual(len(df['columns']), df['n_cols'])
|
||||
self.assertListEqual(df['col_idx'].tolist(), [0, 1, 4])
|
||||
|
||||
def test_data_put_single_var(self):
|
||||
for axis in ["obs", "var"]:
|
||||
endpoint = f"data/{axis}"
|
||||
@@ -302,6 +391,7 @@ class EndPoints(unittest.TestCase):
|
||||
var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}}}
|
||||
result = self.session.put(url, headers=header, json=var_filter)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
result_data = result.json()
|
||||
if axis == "obs":
|
||||
self.assertEqual(len(result_data["obs"][0]), 2)
|
||||
@@ -338,6 +428,7 @@ class EndPoints(unittest.TestCase):
|
||||
}
|
||||
result = self.session.put(url, json=f1)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
result_data1 = result.json()
|
||||
f2 = {
|
||||
"filter": {
|
||||
@@ -353,6 +444,7 @@ class EndPoints(unittest.TestCase):
|
||||
}
|
||||
result = self.session.put(url, json=f2)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
result_data2 = result.json()
|
||||
self.assertNotEqual(result_data1, result_data2)
|
||||
|
||||
@@ -377,6 +469,7 @@ class EndPoints(unittest.TestCase):
|
||||
}
|
||||
result = self.session.put(url, json=f2)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
result_data2 = result.json()
|
||||
self.assertNotEqual(result_data1, result_data2)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user