fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-26 03:28:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Makefile modularity, test targets, and auto-formatting (#1070)

Browse Source 
* Fix Makefile whitespace and .PHONY use

* Fix Makefile filename

* Modularize Makefile into client and server Makefiles

Part of the reason that the Makefile in the root directory is a bit
complicated is that it tries to handle tasks that can be handled
separately in the client and server modules.

This commit pushes some of the make logic specific to each module into
their own makefiles and calls out to those makefiles from that in the
project root.

* Add auto-formatting to client and server modules

One thing that can make linting faster is auto-formatting. This commit
adds the yapf auto-formatting tool to the server module and uses
eslint's "fix" functionality to speed up the linting/formatting process.

* Add yapf for automatic code formatting

* Add a root test target that calls sub-tests

* Apply yapf to python files

* Do not duplicate npm commands, simply pass through

* Update documentation

* Do not shadow reserved word len

* Add general test target

* Fix make call in dev-env

* Use black instead of yapf

* Run flake8 from the root directory

* Revert "Apply yapf to python files"

This reverts commit cdca128a01.

* Apply black to python code

* Resolve lint errors resulting from black format

* Add explanation of server unit tests in dev guidelines
This commit is contained in:
Matt Weiden
2019-12-27 14:43:37 -08:00
committed by GitHub
parent ec79995be8
commit f3015cb9df
37 changed files with 738 additions and 806 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/test/decode_fbs.py
+4 -11
View File
@@ -1,4 +1,3 @@
"""
Code to decode, for testing purposes, the flatbuffer encoded blobs.
This code will need to be updated if fbs/matrix.fbs changes.
@@ -22,20 +21,20 @@ def decode_typed_array(tarr):
TypedArray.TypedArray.Int32Array: Int32Array.Int32Array,
TypedArray.TypedArray.Float32Array: Float32Array.Float32Array,
TypedArray.TypedArray.Float64Array: Float64Array.Float64Array,
TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray
TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray,
}
(u_type, u) = tarr
if u_type == TypedArray.TypedArray.NONE:
return None
TarType = type_map.get(u_type, None)
assert(TarType is not None)
assert TarType is not None
arr = TarType()
arr.Init(u.Bytes, u.Pos)
narr = arr.DataAsNumpy()
if u_type == TypedArray.TypedArray.JSONEncodedArray:
narr = json.loads(narr.tostring().decode('utf-8'))
narr = json.loads(narr.tostring().decode("utf-8"))
return narr
@@ -60,10 +59,4 @@ def decode_matrix_FBS(buf):
cidx = decode_typed_array((df.ColIndexType(), df.ColIndex()))
return {
"n_rows": n_rows,
"n_cols": n_cols,
"columns": decoded_columns,
"col_idx": cidx,
"row_idx": None
}
return {"n_rows": n_rows, "n_cols": n_cols, "columns": decoded_columns, "col_idx": cidx, "row_idx": None}
server/test/test_api.py
+52 -57
View File
@@ -19,7 +19,7 @@ class EndPoints(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.ps = Popen(["cellxgene", "launch", "example-dataset/pbmc3k.h5ad", "--verbose", "--port", "5005"])
cls.ps = Popen(["cellxgene", "launch", "../example-dataset/pbmc3k.h5ad", "--verbose", "--port", "5005"])
session = requests.Session()
for i in range(90):
try:
@@ -68,14 +68,15 @@ class EndPoints(unittest.TestCase):
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 8)
self.assertIsNotNone(df['columns'])
self.assertListEqual(df['col_idx'], [
'pca_0', 'pca_1', 'tsne_0', 'tsne_1', 'umap_0', 'umap_1', 'draw_graph_fr_0', 'draw_graph_fr_1'
])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 8)
self.assertIsNotNone(df["columns"])
self.assertListEqual(
df["col_idx"],
["pca_0", "pca_1", "tsne_0", "tsne_1", "umap_0", "umap_1", "draw_graph_fr_0", "draw_graph_fr_1"],
)
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
def test_bad_filter(self):
endpoint = "data/var"
@@ -91,14 +92,14 @@ class EndPoints(unittest.TestCase):
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 5)
self.assertIsNotNone(df['columns'])
self.assertIsNotNone(df['col_idx'])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 5)
self.assertIsNotNone(df["columns"])
self.assertIsNotNone(df["col_idx"])
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"]
self.assertListEqual(df['col_idx'], [obs_index_col_name, 'n_genes', 'percent_mito', 'n_counts', 'louvain'])
self.assertListEqual(df["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"])
def test_get_annotations_obs_keys_fbs(self):
endpoint = "annotations/obs"
@@ -109,13 +110,13 @@ class EndPoints(unittest.TestCase):
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 2)
self.assertIsNotNone(df['columns'])
self.assertIsNotNone(df['col_idx'])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
self.assertListEqual(df['col_idx'], ['n_genes', 'percent_mito'])
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 2)
self.assertIsNotNone(df["columns"])
self.assertIsNotNone(df["col_idx"])
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
self.assertListEqual(df["col_idx"], ["n_genes", "percent_mito"])
def test_get_annotations_obs_error(self):
endpoint = "annotations/obs"
@@ -162,14 +163,14 @@ class EndPoints(unittest.TestCase):
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 1838)
self.assertEqual(df['n_cols'], 2)
self.assertIsNotNone(df['columns'])
self.assertIsNotNone(df['col_idx'])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
self.assertEqual(df["n_rows"], 1838)
self.assertEqual(df["n_cols"], 2)
self.assertIsNotNone(df["columns"])
self.assertIsNotNone(df["col_idx"])
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
self.assertListEqual(df['col_idx'], [var_index_col_name, 'n_cells'])
self.assertListEqual(df["col_idx"], [var_index_col_name, "n_cells"])
def test_get_annotations_var_keys_fbs(self):
endpoint = "annotations/var"
@@ -180,13 +181,13 @@ class EndPoints(unittest.TestCase):
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 1838)
self.assertEqual(df['n_cols'], 1)
self.assertIsNotNone(df['columns'])
self.assertIsNotNone(df['col_idx'])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
self.assertListEqual(df['col_idx'], ['n_cells'])
self.assertEqual(df["n_rows"], 1838)
self.assertEqual(df["n_cols"], 1)
self.assertIsNotNone(df["columns"])
self.assertIsNotNone(df["col_idx"])
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
self.assertListEqual(df["col_idx"], ["n_cells"])
def test_get_annotations_var_error(self):
endpoint = "annotations/var"
@@ -217,35 +218,29 @@ class EndPoints(unittest.TestCase):
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 1838)
self.assertIsNotNone(df['columns'])
self.assertListEqual(df['col_idx'].tolist(), [])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1838)
self.assertIsNotNone(df["columns"])
self.assertListEqual(df["col_idx"].tolist(), [])
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
def test_data_put_filter_fbs(self):
endpoint = f"data/var"
url = f"{URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
filter = {
"filter": {
"var": {
"index": [0, 1, 4]
}
}
}
filter = {"filter": {"var": {"index": [0, 1, 4]}}}
result = self.session.put(url, headers=header, json=filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 3)
self.assertIsNotNone(df['columns'])
self.assertIsNotNone(df['col_idx'])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
self.assertListEqual(df['col_idx'].tolist(), [0, 1, 4])
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 3)
self.assertIsNotNone(df["columns"])
self.assertIsNotNone(df["col_idx"])
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
self.assertListEqual(df["col_idx"].tolist(), [0, 1, 4])
def test_data_put_single_var(self):
endpoint = f"data/var"
server/test/test_fbs.py
+16 -27
View File
@@ -32,7 +32,7 @@ class FbsTests(unittest.TestCase):
for i in range(0, len(d["columns"])):
self.assertEqual(len(d["columns"][i]), dims[0])
self.assertIsInstance(d["columns"][i], expected_types[i][0])
if (expected_types[i][1] is not None):
if expected_types[i][1] is not None:
self.assertEqual(d["columns"][i].dtype, expected_types[i][1])
if expected_column_idx is not None:
self.assertSetEqual(set(expected_column_idx), set(d["col_idx"]))
@@ -40,48 +40,37 @@ class FbsTests(unittest.TestCase):
def test_encode_DataFrame(self):
df = pd.DataFrame(
data={
'a': np.zeros((10,), dtype=np.float32),
'b': np.ones((10,), dtype=np.int64),
'c': np.array([i for i in range(0, 10)], dtype=np.uint16),
'd': pd.Series(['x', 'y', 'z', 'x', 'y', 'z', 'a', 'x', 'y', 'z'], dtype='category')
})
expected_types = (
(np.ndarray, np.float32),
(np.ndarray, np.int32),
(np.ndarray, np.uint32),
(list, None)
"a": np.zeros((10,), dtype=np.float32),
"b": np.ones((10,), dtype=np.int64),
"c": np.array([i for i in range(0, 10)], dtype=np.uint16),
"d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"),
}
)
expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), (np.ndarray, np.uint32), (list, None))
fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
self.fbs_checks(fbs, (10, 4), expected_types, ['a', 'b', 'c', 'd'])
self.fbs_checks(fbs, (10, 4), expected_types, ["a", "b", "c", "d"])
def test_encode_ndarray(self):
arr = np.zeros((3, 2), dtype=np.float32)
expected_types = (
(np.ndarray, np.float32),
(np.ndarray, np.float32),
(np.ndarray, np.float32)
)
expected_types = ((np.ndarray, np.float32), (np.ndarray, np.float32), (np.ndarray, np.float32))
fbs = encode_matrix_fbs(matrix=arr, row_idx=None, col_idx=None)
self.fbs_checks(fbs, (3, 2), expected_types, None)
def test_encode_sparse(self):
csc = sparse.csc_matrix(np.array([[0, 1, 2], [3, 0, 4]]))
expected_types = (
(np.ndarray, np.int32),
(np.ndarray, np.int32),
(np.ndarray, np.int32)
)
expected_types = ((np.ndarray, np.int32), (np.ndarray, np.int32), (np.ndarray, np.int32))
fbs = encode_matrix_fbs(matrix=csc, row_idx=None, col_idx=None)
self.fbs_checks(fbs, (2, 3), expected_types, None)
def test_roundtrip(self):
dfSrc = pd.DataFrame(
data={
'a': np.zeros((10,), dtype=np.float32),
'b': np.ones((10,), dtype=np.int64),
'c': np.array([i for i in range(0, 10)], dtype=np.uint16),
'd': pd.Series(['x', 'y', 'z', 'x', 'y', 'z', 'a', 'x', 'y', 'z'], dtype='category')
})
"a": np.zeros((10,), dtype=np.float32),
"b": np.ones((10,), dtype=np.int64),
"c": np.array([i for i in range(0, 10)], dtype=np.uint16),
"d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"),
}
)
dfDst = decode_matrix_fbs(encode_matrix_fbs(matrix=dfSrc, col_idx=dfSrc.columns))
self.assertEqual(dfSrc.shape, dfDst.shape)
self.assertEqual(set(dfSrc.columns), set(dfDst.columns))
server/test/test_matrix_proxy.py
+39 -95
View File
@@ -7,9 +7,10 @@ class NdArrayProxyView(MatrixProxyView):
"""
Fake test class for matrix proxy - wraps ndarray
"""
@classmethod
def __supports__(cls):
return ('numpy.ndarray', )
return ("numpy.ndarray",)
class MatrixProxyViewTest(unittest.TestCase):
@@ -41,18 +42,17 @@ class MatrixProxyViewTest(unittest.TestCase):
def test_toarray(self):
n = np.arange(15, dtype=np.float32).reshape((3, 5))
mp = MatrixProxy.create(n)
self.assertTrue(np.all(mp.toarray() == [
[0., 1., 2., 3., 4.],
[5., 6., 7., 8., 9.],
[10., 11., 12., 13., 14.]
]))
self.assertTrue(np.all(mp.T.toarray() == [
[0., 5., 10.],
[1., 6., 11.],
[2., 7., 12.],
[3., 8., 13.],
[4., 9., 14.]
]))
self.assertTrue(
np.all(
mp.toarray() == [[0.0, 1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0, 9.0], [10.0, 11.0, 12.0, 13.0, 14.0]]
)
)
self.assertTrue(
np.all(
mp.T.toarray()
== [[0.0, 5.0, 10.0], [1.0, 6.0, 11.0], [2.0, 7.0, 12.0], [3.0, 8.0, 13.0], [4.0, 9.0, 14.0]]
)
)
def test_indexing(self):
"""
@@ -95,47 +95,19 @@ class MatrixProxyViewTest(unittest.TestCase):
# slice, slice
self.assertTrue(np.all(mp[1:3, 2:4].toarray() == [
[7, 8],
[12, 13]
]))
self.assertTrue(np.all(mp[:3, :4].toarray() == [
[0., 1., 2., 3.],
[5., 6., 7., 8.],
[10., 11., 12., 13.]
]))
self.assertTrue(np.all(mp[::-1, ::-1].toarray() == [
[14, 13, 12, 11, 10],
[9, 8, 7, 6, 5],
[4, 3, 2, 1, 0]
]))
self.assertTrue(np.all(mp[::-2, ::-2].toarray() == [
[14, 12, 10],
[4, 2, 0]
]))
self.assertTrue(np.all(mp[1:3, 2:4].toarray() == [[7, 8], [12, 13]]))
self.assertTrue(
np.all(mp[:3, :4].toarray() == [[0.0, 1.0, 2.0, 3.0], [5.0, 6.0, 7.0, 8.0], [10.0, 11.0, 12.0, 13.0]])
)
self.assertTrue(np.all(mp[::-1, ::-1].toarray() == [[14, 13, 12, 11, 10], [9, 8, 7, 6, 5], [4, 3, 2, 1, 0]]))
self.assertTrue(np.all(mp[::-2, ::-2].toarray() == [[14, 12, 10], [4, 2, 0]]))
self.assertTrue(np.all(mp.T[2:4, 1:3].toarray() == [
[7, 12],
[8, 13]
]))
self.assertTrue(np.all(mp.T[:4, :3].toarray() == [
[0, 5, 10],
[1, 6, 11],
[2, 7, 12],
[3, 8, 13]
]))
self.assertTrue(np.all(mp.T[::-1, ::-1].toarray() == [
[14, 9, 4],
[13, 8, 3],
[12, 7, 2],
[11, 6, 1],
[10, 5, 0]
]))
self.assertTrue(np.all(mp.T[::-2, ::-2].toarray() == [
[14, 4],
[12, 2],
[10, 0]
]))
self.assertTrue(np.all(mp.T[2:4, 1:3].toarray() == [[7, 12], [8, 13]]))
self.assertTrue(np.all(mp.T[:4, :3].toarray() == [[0, 5, 10], [1, 6, 11], [2, 7, 12], [3, 8, 13]]))
self.assertTrue(
np.all(mp.T[::-1, ::-1].toarray() == [[14, 9, 4], [13, 8, 3], [12, 7, 2], [11, 6, 1], [10, 5, 0]])
)
self.assertTrue(np.all(mp.T[::-2, ::-2].toarray() == [[14, 4], [12, 2], [10, 0]]))
def test_repeated_indexing(self):
"""
@@ -149,31 +121,15 @@ class MatrixProxyViewTest(unittest.TestCase):
self.assertEqual(mp[0][1], 1)
self.assertEqual(mp.T[0][1], 5)
self.assertTrue(np.all(mp[0::-1, ::-1][0, 2:4].toarray() == [
2, 1
]))
self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 1:3:1].toarray() == [
2, 3
]))
self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 2:0:-1].toarray() == [
3, 2
]))
self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 1:3:1].toarray() == [
3, 2
]))
self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 2:0:-1].toarray() == [
2, 3
]))
self.assertTrue(np.all(mp[0::-1, ::-1][0, 2:4].toarray() == [2, 1]))
self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 1:3:1].toarray() == [2, 3]))
self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 2:0:-1].toarray() == [3, 2]))
self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 1:3:1].toarray() == [3, 2]))
self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 2:0:-1].toarray() == [2, 3]))
self.assertTrue(np.all(mp.T[::-1, 0::-1][2:4, 0].toarray() == [
2, 1
]))
self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 1:3:1].toarray() == [
10
]))
self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 2:0:-1].toarray() == [
10
]))
self.assertTrue(np.all(mp.T[::-1, 0::-1][2:4, 0].toarray() == [2, 1]))
self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 1:3:1].toarray() == [10]))
self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 2:0:-1].toarray() == [10]))
self.assertTrue(np.all(mp.T[0::-1, 5:1:-1][0, 1:3:1].toarray() == []))
self.assertTrue(np.all(mp.T[0::-1, 5:1:-1][0, 2:0:-1].toarray() == []))
@@ -190,20 +146,12 @@ class MatrixProxyViewTest(unittest.TestCase):
self.assertEqual(mp[0, 0], 0)
# drop 1 dimension, to an array
self.assertTrue(np.all(mp[0, :].toarray() == [
0, 1, 2, 3, 4
]))
self.assertTrue(np.all(mp[:, 0].toarray() == [
0, 5, 10
]))
self.assertTrue(np.all(mp[0, :].toarray() == [0, 1, 2, 3, 4]))
self.assertTrue(np.all(mp[:, 0].toarray() == [0, 5, 10]))
# with .T
self.assertTrue(np.all(mp[0:2].T[-1:].toarray() == [
[4, 9]
]))
self.assertTrue(np.all(mp[0:2].T[-1].toarray() == [
4, 9
]))
self.assertTrue(np.all(mp[0:2].T[-1:].toarray() == [[4, 9]]))
self.assertTrue(np.all(mp[0:2].T[-1].toarray() == [4, 9]))
def test_iter(self):
"""
@@ -214,17 +162,13 @@ class MatrixProxyViewTest(unittest.TestCase):
rows = [r for r in mp]
self.assertEqual(len(rows), 3)
self.assertTrue(np.all(rows[0].toarray() == [
0, 1, 2, 3, 4
]))
self.assertTrue(np.all(rows[0].toarray() == [0, 1, 2, 3, 4]))
for i, r in enumerate(rows):
self.assertTrue(np.all(mp[i].toarray() == r.toarray()))
cols = [c for c in mp.T]
self.assertEqual(len(cols), 5)
self.assertTrue(np.all(cols[0].toarray() == [
0, 5, 10
]))
self.assertTrue(np.all(cols[0].toarray() == [0, 5, 10]))
for i, c in enumerate(cols):
self.assertTrue(np.all(mp.T[i].toarray() == c.toarray()))
server/test/test_nan_rest.py
+1 -3
View File
@@ -20,9 +20,7 @@ class WithNaNs(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.ps = Popen(
["cellxgene", "launch", "server/test/test_datasets/nan.h5ad", "--verbose", "--port", "5006"]
)
cls.ps = Popen(["cellxgene", "launch", "test/test_datasets/nan.h5ad", "--verbose", "--port", "5006"])
session = requests.Session()
for i in range(90):
try:
server/test/test_nan_scanpy_engine.py
+4 -11
View File
@@ -21,12 +21,12 @@ class NaNTest(unittest.TestCase):
}
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UserWarning)
self.data = ScanpyEngine(DataLocator("server/test/test_datasets/nan.h5ad"), self.args)
self.data = ScanpyEngine(DataLocator("test/test_datasets/nan.h5ad"), self.args)
self.data._create_schema()
def test_load(self):
with self.assertWarns(UserWarning):
ScanpyEngine(DataLocator("server/test/test_datasets/nan.h5ad"), self.args)
ScanpyEngine(DataLocator("test/test_datasets/nan.h5ad"), self.args)
def test_init(self):
self.assertEqual(self.data.cell_count, 100)
@@ -44,11 +44,7 @@ class NaNTest(unittest.TestCase):
with pytest.raises(FilterError):
self.data.data_frame_to_fbs_matrix("an erroneous filter", "var")
with pytest.raises(FilterError):
filter_ = {
"filter": {
"obs": {"index": [1, 99, [200, 300]]}
}
}
filter_ = {"filter": {"obs": {"index": [1, 99, [200, 300]]}}}
self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
def test_dataframe_obs_not_implemented(self):
@@ -59,10 +55,7 @@ class NaNTest(unittest.TestCase):
def test_annotation(self):
annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("obs"))
obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
self.assertEqual(
annotations["col_idx"],
[obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"]
)
self.assertEqual(annotations["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"])
self.assertEqual(annotations["n_rows"], 100)
self.assertTrue(math.isnan(annotations["columns"][2][0]))
server/test/test_scanpy_engine.py
+26 -48
View File
@@ -18,15 +18,17 @@ Test the scanpy engine using the pbmc3k data set.
"""
@parameterized_class(("data_locator", "backed"), [
("example-dataset/pbmc3k.h5ad", False),
("server/test/test_datasets/pbmc3k-CSC-gz.h5ad", False),
("server/test/test_datasets/pbmc3k-CSR-gz.h5ad", False),
("example-dataset/pbmc3k.h5ad", True),
("server/test/test_datasets/pbmc3k-CSC-gz.h5ad", True),
("server/test/test_datasets/pbmc3k-CSR-gz.h5ad", True),
])
@parameterized_class(
("data_locator", "backed"),
[
("../example-dataset/pbmc3k.h5ad", False),
("test/test_datasets/pbmc3k-CSC-gz.h5ad", False),
("test/test_datasets/pbmc3k-CSR-gz.h5ad", False),
("../example-dataset/pbmc3k.h5ad", True),
("test/test_datasets/pbmc3k-CSC-gz.h5ad", True),
("test/test_datasets/pbmc3k-CSR-gz.h5ad", True),
],
)
class EngineTest(unittest.TestCase):
def setUp(self):
args = {
@@ -36,7 +38,7 @@ class EngineTest(unittest.TestCase):
"var_names": None,
"diffexp_lfc_cutoff": 0.01,
"layout_file": None,
"backed": self.backed
"backed": self.backed,
}
self.data = ScanpyEngine(DataLocator(self.data_locator), args)
@@ -64,11 +66,7 @@ class EngineTest(unittest.TestCase):
self.data._validate_data_types()
def test_filter_idx(self):
filter_ = {
"filter": {
"var": {"index": [1, 99, [200, 300]]}
}
}
filter_ = {"filter": {"var": {"index": [1, 99, [200, 300]]}}}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(data["n_rows"], 2638)
@@ -76,14 +74,7 @@ class EngineTest(unittest.TestCase):
def test_filter_complex(self):
filter_ = {
"filter": {
"var": {
"annotation_value": [
{"name": "n_cells", "min": 10}
],
"index": [1, 99, [200, 300]]
}
}
"filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 10}], "index": [1, 99, [200, 300]]}}
}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs)
@@ -101,16 +92,14 @@ class EngineTest(unittest.TestCase):
def test_schema_produces_error(self):
self.data.data.obs["time"] = pd.Series(
list([time.time() for i in range(self.data.cell_count)]),
dtype="datetime64[ns]",
list([time.time() for i in range(self.data.cell_count)]), dtype="datetime64[ns]",
)
with pytest.raises(TypeError):
self.data._create_schema()
def test_config(self):
self.assertEqual(
self.data.features["layout"]["obs"],
{"available": True, "interactiveLimit": 50000},
self.data.features["layout"]["obs"], {"available": True, "interactiveLimit": 50000},
)
def test_layout(self):
@@ -131,14 +120,13 @@ class EngineTest(unittest.TestCase):
self.assertEqual(annotations["n_cols"], 5)
obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"]
self.assertEqual(
annotations["col_idx"],
[obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
annotations["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
)
fbs = self.data.annotation_to_fbs_matrix("var")
annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations['n_rows'], 1838)
self.assertEqual(annotations['n_cols'], 2)
self.assertEqual(annotations["n_rows"], 1838)
self.assertEqual(annotations["n_cols"], 2)
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells"])
@@ -146,13 +134,13 @@ class EngineTest(unittest.TestCase):
fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"])
annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations["n_rows"], 2638)
self.assertEqual(annotations['n_cols'], 2)
self.assertEqual(annotations["n_cols"], 2)
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name])
annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations['n_rows'], 1838)
self.assertEqual(annotations['n_cols'], 1)
self.assertEqual(annotations["n_rows"], 1838)
self.assertEqual(annotations["n_cols"], 1)
def test_annotation_put(self):
with self.assertRaises(DisabledFeatureError):
@@ -177,27 +165,19 @@ class EngineTest(unittest.TestCase):
self.data.data_frame_to_fbs_matrix(None, "obs")
def test_filtered_data_frame(self):
filter_ = {
"filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 100}]}}
}
filter_ = {"filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 100}]}}}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(data["n_rows"], 2638)
self.assertEqual(data["n_cols"], 1040)
filter_ = {
"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}
}
filter_ = {"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}}
with self.assertRaises(FilterError):
self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
def test_data_named_gene(self):
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
filter_ = {
"filter": {
"var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]}
}
}
filter_ = {"filter": {"var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]}}}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(data["n_rows"], 2638)
@@ -205,9 +185,7 @@ class EngineTest(unittest.TestCase):
self.assertEqual(data["col_idx"], [4])
filter_ = {
"filter": {
"var": {"annotation_value": [{"name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"]}]}
}
"filter": {"var": {"annotation_value": [{"name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"]}]}}
}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs)
server/test/test_scanpy_engine_data_load.py
+5 -3
View File
@@ -10,8 +10,9 @@ class DataLoadEngineTest(unittest.TestCase):
"""
Test file loading, including deferred loading/update.
"""
def setUp(self):
self.data_file = DataLocator("example-dataset/pbmc3k.h5ad")
self.data_file = DataLocator("../example-dataset/pbmc3k.h5ad")
self.data = ScanpyEngine()
def test_init(self):
@@ -29,7 +30,7 @@ class DataLoadEngineTest(unittest.TestCase):
"annotations_output_dir": None,
"backed": False,
"diffexp_may_be_slow": False,
"disable_diffexp": False
"disable_diffexp": False,
}
self.data.update(args=args)
self.assertEqual(args, self.data.config)
@@ -60,6 +61,7 @@ class DataLocatorEngineTest(unittest.TestCase):
"""
Test various types of data locators we expect to consume
"""
def setUp(self):
self.args = {
"layout": ["umap"],
@@ -76,7 +78,7 @@ class DataLocatorEngineTest(unittest.TestCase):
self.assertEqual(data.gene_count, 1838)
def test_posix_file(self):
locator = DataLocator("example-dataset/pbmc3k.h5ad")
locator = DataLocator("../example-dataset/pbmc3k.h5ad")
data = ScanpyEngine(locator, self.args)
self.stdAsserts(data)
server/test/test_writable_annotation.py
+49 -50
View File
@@ -25,9 +25,9 @@ class WritableAnnotationTest(unittest.TestCase):
"diffexp_lfc_cutoff": 0.01,
"annotations": True,
"annotations_file": self.annotations_file,
"annotations_output_dir": None
"annotations_output_dir": None,
}
self.data = ScanpyEngine(DataLocator("example-dataset/pbmc3k.h5ad"), args)
self.data = ScanpyEngine(DataLocator("../example-dataset/pbmc3k.h5ad"), args)
def tearDown(self):
shutil.rmtree(self.tmpDir)
@@ -40,9 +40,7 @@ class WritableAnnotationTest(unittest.TestCase):
# verify that the expected errors are generated
n_rows = self.data.data.obs.shape[0]
fbs_bad = self.make_fbs({
'louvain': pd.Series(['undefined' for l in range(0, n_rows)], dtype='category')
})
fbs_bad = self.make_fbs({"louvain": pd.Series(["undefined" for l in range(0, n_rows)], dtype="category")})
# ensure attempt to change VAR annotation
with self.assertRaises(ValueError):
@@ -55,32 +53,36 @@ class WritableAnnotationTest(unittest.TestCase):
def test_write_to_file(self):
# verify the file is written as expected
n_rows = self.data.data.obs.shape[0]
fbs = self.make_fbs({
'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'),
'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category')
})
fbs = self.make_fbs(
{
"cat_A": pd.Series(["label_A" for l in range(0, n_rows)], dtype="category"),
"cat_B": pd.Series(["label_B" for l in range(0, n_rows)], dtype="category"),
}
)
res = self.data.annotation_put_fbs("obs", fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
self.assertTrue(path.exists(self.annotations_file))
df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment='#')
df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment="#")
self.assertEqual(df.shape, (n_rows, 2))
self.assertEqual(set(df.columns), set(['cat_A', 'cat_B']))
self.assertEqual(set(df.columns), set(["cat_A", "cat_B"]))
self.assertTrue(self.data.original_obs_index.equals(df.index))
self.assertTrue(np.all(df['cat_A'] == ['label_A' for l in range(0, n_rows)]))
self.assertTrue(np.all(df['cat_B'] == ['label_B' for l in range(0, n_rows)]))
self.assertTrue(np.all(df["cat_A"] == ["label_A" for l in range(0, n_rows)]))
self.assertTrue(np.all(df["cat_B"] == ["label_B" for l in range(0, n_rows)]))
# verify complete overwrite on second attempt, AND rotation occurs
fbs = self.make_fbs({
'cat_A': pd.Series(['label_A1' for l in range(0, n_rows)], dtype='category'),
'cat_C': pd.Series(['label_C' for l in range(0, n_rows)], dtype='category')
})
fbs = self.make_fbs(
{
"cat_A": pd.Series(["label_A1" for l in range(0, n_rows)], dtype="category"),
"cat_C": pd.Series(["label_C" for l in range(0, n_rows)], dtype="category"),
}
)
res = self.data.annotation_put_fbs("obs", fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
self.assertTrue(path.exists(self.annotations_file))
df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment='#')
self.assertEqual(set(df.columns), set(['cat_A', 'cat_C']))
self.assertTrue(np.all(df['cat_A'] == ['label_A1' for l in range(0, n_rows)]))
self.assertTrue(np.all(df['cat_C'] == ['label_C' for l in range(0, n_rows)]))
df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment="#")
self.assertEqual(set(df.columns), set(["cat_A", "cat_C"]))
self.assertTrue(np.all(df["cat_A"] == ["label_A1" for l in range(0, n_rows)]))
self.assertTrue(np.all(df["cat_C"] == ["label_C" for l in range(0, n_rows)]))
# rotation
name, ext = path.splitext(self.annotations_file)
@@ -92,10 +94,12 @@ class WritableAnnotationTest(unittest.TestCase):
def test_file_rotation_to_max_9(self):
# verify we stop rotation at 9
n_rows = self.data.data.obs.shape[0]
fbs = self.make_fbs({
'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'),
'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category')
})
fbs = self.make_fbs(
{
"cat_A": pd.Series(["label_A" for l in range(0, n_rows)], dtype="category"),
"cat_B": pd.Series(["label_B" for l in range(0, n_rows)], dtype="category"),
}
)
for i in range(0, 11):
res = self.data.annotation_put_fbs("obs", fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
@@ -111,10 +115,12 @@ class WritableAnnotationTest(unittest.TestCase):
# GET (annotation_to_fbs_matrix)
n_rows = self.data.data.obs.shape[0]
fbs = self.make_fbs({
'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'),
'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category')
})
fbs = self.make_fbs(
{
"cat_A": pd.Series(["label_A" for l in range(0, n_rows)], dtype="category"),
"cat_B": pd.Series(["label_B" for l in range(0, n_rows)], dtype="category"),
}
)
# put
res = self.data.annotation_put_fbs("obs", fbs)
@@ -128,28 +134,21 @@ class WritableAnnotationTest(unittest.TestCase):
self.assertEqual(annotations["n_rows"], n_rows)
self.assertEqual(annotations["n_cols"], 7)
self.assertIsNone(annotations["row_idx"])
self.assertEqual(annotations["col_idx"], [
obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain", "cat_A", "cat_B"
])
self.assertEqual(
annotations["col_idx"],
[obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain", "cat_A", "cat_B"],
)
col_idx = annotations["col_idx"]
self.assertEqual(annotations["columns"][col_idx.index('cat_A')], [
'label_A' for l in range(0, n_rows)
])
self.assertEqual(annotations["columns"][col_idx.index('cat_B')], [
'label_B' for l in range(0, n_rows)
])
self.assertEqual(annotations["columns"][col_idx.index("cat_A")], ["label_A" for l in range(0, n_rows)])
self.assertEqual(annotations["columns"][col_idx.index("cat_B")], ["label_B" for l in range(0, n_rows)])
# verify the schema was updated
all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]}
self.assertEqual(all_col_schema["cat_A"], {
"name": "cat_A",
"type": "categorical",
"categories": ["label_A"],
"writable": True
})
self.assertEqual(all_col_schema["cat_B"], {
"name": "cat_B",
"type": "categorical",
"categories": ["label_B"],
"writable": True
})
self.assertEqual(
all_col_schema["cat_A"],
{"name": "cat_A", "type": "categorical", "categories": ["label_A"], "writable": True},
)
self.assertEqual(
all_col_schema["cat_B"],
{"name": "cat_B", "type": "categorical", "categories": ["label_B"], "writable": True},
)