server refactor (#1140)

This PR contains a refactoring to make adding new features easier.

The new features include supporting the tiledb format, and the multi dataset application.

The refactoring includes

Simplifying the directory structure and files.
a class structure to handle annotations (currently one type: AnnotationsLocalFile).
a class to handle application configuration
a class structure to handle matrix data (currently AnndataAdaptor and CxgAdaptor). CxgAdaptor uses tiledb.
Algorithms that were previously dependent on the scanpy anndata object are now generalized to work with an abstract interface.
The multi dataset option is not fully supported yet, and so the option to use it is hidden.
Use "cli launch --dataroot ..."
To access this feature.

All combinations of app single dataset/ app multi dataset and AnndataAdaptor/CxgAdaptor work with all the features, such as annotations, ontologies, diffexp.
This commit is contained in:
bmccandless
2020-02-19 10:22:35 -08:00
committed by GitHub
parent 349c413d8b
commit 907cc634f5
116 changed files with 2697 additions and 3252 deletions
+8 -8
View File
@@ -2,17 +2,17 @@
Code to decode, for testing purposes, the flatbuffer encoded blobs.
This code will need to be updated if fbs/matrix.fbs changes.
For more information, see fbs/matrix.fbs and server/app/util/fbs/
For more information, see fbs/matrix.fbs and server/data_common/fbs/
"""
import json
import server.app.util.fbs.NetEncoding.TypedArray as TypedArray
import server.app.util.fbs.NetEncoding.Matrix as Matrix
import server.app.util.fbs.NetEncoding.Int32Array as Int32Array
import server.app.util.fbs.NetEncoding.Uint32Array as Uint32Array
import server.app.util.fbs.NetEncoding.Float32Array as Float32Array
import server.app.util.fbs.NetEncoding.Float64Array as Float64Array
import server.app.util.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
import server.data_common.fbs.NetEncoding.TypedArray as TypedArray
import server.data_common.fbs.NetEncoding.Matrix as Matrix
import server.data_common.fbs.NetEncoding.Int32Array as Int32Array
import server.data_common.fbs.NetEncoding.Uint32Array as Uint32Array
import server.data_common.fbs.NetEncoding.Float32Array as Float32Array
import server.data_common.fbs.NetEncoding.Float64Array as Float64Array
import server.data_common.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
def decode_typed_array(tarr):
@@ -9,12 +9,12 @@ from parameterized import parameterized_class
import numpy as np
import pandas as pd
from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
from server.app.util.errors import FilterError, DisabledFeatureError
from server.app.util.data_locator import DataLocator
from server.data_anndata.anndata_adaptor import AnndataAdaptor
from server.common.errors import FilterError
from server.common.data_locator import DataLocator
"""
Test the scanpy engine using the pbmc3k data set.
Test the anndata adaptor using the pbmc3k data set.
"""
@@ -29,7 +29,7 @@ Test the scanpy engine using the pbmc3k data set.
("test/test_datasets/pbmc3k-CSR-gz.h5ad", True),
],
)
class EngineTest(unittest.TestCase):
class AdaptorTest(unittest.TestCase):
def setUp(self):
args = {
"layout": ["umap"],
@@ -37,10 +37,9 @@ class EngineTest(unittest.TestCase):
"obs_names": None,
"var_names": None,
"diffexp_lfc_cutoff": 0.01,
"layout_file": None,
"backed": self.backed,
"anndata_backed": self.backed,
}
self.data = ScanpyEngine(DataLocator(self.data_locator), args)
self.data = AnndataAdaptor(DataLocator(self.data_locator), args)
def test_init(self):
self.assertEqual(self.data.cell_count, 2638)
@@ -56,7 +55,7 @@ class EngineTest(unittest.TestCase):
self.assertIn(var_index_col_name, self.data.data.var)
self.assertEqual(list(self.data.data.var.index), list(range(1838)))
@pytest.mark.filterwarnings("ignore:Scanpy data matrix")
@pytest.mark.filterwarnings("ignore:Anndata data matrix")
def test_data_type(self):
# don't run the test on the more exotic data types, as they don't
# support the astype() interface (used by this test, but not underlying app)
@@ -98,9 +97,7 @@ class EngineTest(unittest.TestCase):
self.data._create_schema()
def test_config(self):
self.assertEqual(
self.data.features["layout"]["obs"], {"available": True, "interactiveLimit": 50000},
)
self.assertEqual(self.data.get_features()["layout_obs"].available, True)
def test_layout(self):
fbs = self.data.layout_to_fbs_matrix()
@@ -142,10 +139,6 @@ class EngineTest(unittest.TestCase):
self.assertEqual(annotations["n_rows"], 1838)
self.assertEqual(annotations["n_cols"], 1)
def test_annotation_put(self):
with self.assertRaises(DisabledFeatureError):
self.data.annotation_put_fbs(None, "obs")
def test_diffexp_topN(self):
f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
@@ -1,49 +1,20 @@
import unittest
import json
from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
from server.app.util.errors import DriverError
from server.app.util.data_locator import DataLocator
from server.data_anndata.anndata_adaptor import AnndataAdaptor
from server.common.data_locator import DataLocator
class DataLoadEngineTest(unittest.TestCase):
class DataLoadAdaptorTest(unittest.TestCase):
"""
Test file loading, including deferred loading/update.
"""
def setUp(self):
self.data_file = DataLocator("../example-dataset/pbmc3k.h5ad")
self.data = ScanpyEngine()
def test_init(self):
self.assertIsNone(self.data.data)
def test_delayed_load_args(self):
args = {
"layout": ["tsne"],
"max_category_items": 1000,
"obs_names": "foo",
"var_names": "bar",
"diffexp_lfc_cutoff": 0.1,
"annotations": False,
"annotations_file": None,
"annotations_output_dir": None,
"backed": False,
"diffexp_may_be_slow": False,
"disable_diffexp": False,
"annotations_cell_ontology_enabled": False,
"annotations_cell_ontology_obopath": None,
"annotations_cell_ontology_terms": None,
}
self.data.update(args=args)
self.assertEqual(args, self.data.config)
def test_requires_data(self):
with self.assertRaises(DriverError):
self.data._create_schema()
self.data = AnndataAdaptor(self.data_file)
def test_delayed_load_data(self):
self.data.update(data_locator=self.data_file)
self.data._create_schema()
self.assertEqual(self.data.cell_count, 2638)
self.assertEqual(self.data.gene_count, 1838)
@@ -51,7 +22,6 @@ class DataLoadEngineTest(unittest.TestCase):
self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
def test_diffexp_topN(self):
self.data.update(data_locator=self.data_file)
f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
@@ -60,7 +30,7 @@ class DataLoadEngineTest(unittest.TestCase):
self.assertEqual(len(result), 20)
class DataLocatorEngineTest(unittest.TestCase):
class DataLocatorAdaptorTest(unittest.TestCase):
"""
Test various types of data locators we expect to consume
"""
@@ -82,17 +52,17 @@ class DataLocatorEngineTest(unittest.TestCase):
def test_posix_file(self):
locator = DataLocator("../example-dataset/pbmc3k.h5ad")
data = ScanpyEngine(locator, self.args)
data = AnndataAdaptor(locator, self.args)
self.stdAsserts(data)
def test_url_https(self):
url = "https://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/example-dataset/pbmc3k.h5ad"
locator = DataLocator(url)
data = ScanpyEngine(locator, self.args)
data = AnndataAdaptor(locator, self.args)
self.stdAsserts(data)
def test_url_http(self):
url = "http://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/example-dataset/pbmc3k.h5ad"
locator = DataLocator(url)
data = ScanpyEngine(locator, self.args)
data = AnndataAdaptor(locator, self.args)
self.stdAsserts(data)
+97 -55
View File
@@ -7,50 +7,18 @@ import requests
import decode_fbs
LOCAL_URL = "http://127.0.0.1:5005/"
VERSION = "v0.2"
URL_BASE = f"{LOCAL_URL}api/{VERSION}/"
BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
class EndPoints(unittest.TestCase):
"""Test Case for endpoints"""
@classmethod
def setUpClass(cls):
cls.ps = Popen(
[
"cellxgene",
"--no-upgrade-check",
"launch",
"../example-dataset/pbmc3k.h5ad",
"--verbose",
"--port",
"5005",
]
)
session = requests.Session()
for i in range(90):
try:
result = session.get(f"{URL_BASE}schema")
cls.schema = result.json()
except requests.exceptions.ConnectionError:
time.sleep(1)
@classmethod
def tearDownClass(cls):
try:
cls.ps.terminate()
except ProcessLookupError:
pass
class EndPoints(object):
def setUp(self):
self.session = requests.Session()
def test_initialize(self):
endpoint = "schema"
url = f"{URL_BASE}{endpoint}"
url = f"{self.URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
@@ -61,18 +29,18 @@ class EndPoints(unittest.TestCase):
def test_config(self):
endpoint = "config"
url = f"{URL_BASE}{endpoint}"
url = f"{self.URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertIn("library_versions", result_data["config"])
self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k")
self.assertEqual(len(result_data["config"]["features"]), 4)
self.assertEqual(len(result_data["config"]["features"]), 3)
def test_get_layout_fbs(self):
endpoint = "layout/obs"
url = f"{URL_BASE}{endpoint}"
url = f"{self.URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
@@ -81,22 +49,22 @@ class EndPoints(unittest.TestCase):
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 8)
self.assertIsNotNone(df["columns"])
self.assertListEqual(
df["col_idx"],
["pca_0", "pca_1", "tsne_0", "tsne_1", "umap_0", "umap_1", "draw_graph_fr_0", "draw_graph_fr_1"],
self.assertSetEqual(
set(df["col_idx"]),
set(["pca_0", "pca_1", "tsne_0", "tsne_1", "umap_0", "umap_1", "draw_graph_fr_0", "draw_graph_fr_1"]),
)
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
def test_bad_filter(self):
endpoint = "data/var"
url = f"{URL_BASE}{endpoint}"
url = f"{self.URL_BASE}{endpoint}"
result = self.session.put(url, json=BAD_FILTER)
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
def test_get_annotations_obs_fbs(self):
endpoint = "annotations/obs"
url = f"{URL_BASE}{endpoint}"
url = f"{self.URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
@@ -114,7 +82,7 @@ class EndPoints(unittest.TestCase):
def test_get_annotations_obs_keys_fbs(self):
endpoint = "annotations/obs"
query = "annotation-name=n_genes&annotation-name=percent_mito"
url = f"{URL_BASE}{endpoint}?{query}"
url = f"{self.URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
@@ -131,13 +99,13 @@ class EndPoints(unittest.TestCase):
def test_get_annotations_obs_error(self):
endpoint = "annotations/obs"
query = "annotation-name=notakey"
url = f"{URL_BASE}{endpoint}?{query}"
url = f"{self.URL_BASE}{endpoint}?{query}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
def test_diff_exp(self):
endpoint = "diffexp/obs"
url = f"{URL_BASE}{endpoint}"
url = f"{self.URL_BASE}{endpoint}"
params = {
"mode": "topN",
"set1": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["NK cells"]}]}}},
@@ -152,7 +120,7 @@ class EndPoints(unittest.TestCase):
def test_diff_exp_indices(self):
endpoint = "diffexp/obs"
url = f"{URL_BASE}{endpoint}"
url = f"{self.URL_BASE}{endpoint}"
params = {
"mode": "topN",
"count": 10,
@@ -167,7 +135,7 @@ class EndPoints(unittest.TestCase):
def test_get_annotations_var_fbs(self):
endpoint = "annotations/var"
url = f"{URL_BASE}{endpoint}"
url = f"{self.URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
@@ -185,7 +153,7 @@ class EndPoints(unittest.TestCase):
def test_get_annotations_var_keys_fbs(self):
endpoint = "annotations/var"
query = "annotation-name=n_cells"
url = f"{URL_BASE}{endpoint}?{query}"
url = f"{self.URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
@@ -202,27 +170,27 @@ class EndPoints(unittest.TestCase):
def test_get_annotations_var_error(self):
endpoint = "annotations/var"
query = "annotation-name=notakey"
url = f"{URL_BASE}{endpoint}?{query}"
url = f"{self.URL_BASE}{endpoint}?{query}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
def test_data_mimetype_error(self):
endpoint = f"data/var"
header = {"Accept": "xxx"}
url = f"{URL_BASE}{endpoint}"
url = f"{self.URL_BASE}{endpoint}"
result = self.session.put(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.NOT_ACCEPTABLE)
def test_fbs_default(self):
endpoint = f"data/var"
url = f"{URL_BASE}{endpoint}"
url = f"{self.URL_BASE}{endpoint}"
result = self.session.put(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
def test_data_put_fbs(self):
endpoint = f"data/var"
url = f"{URL_BASE}{endpoint}"
url = f"{self.URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.session.put(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
@@ -237,7 +205,7 @@ class EndPoints(unittest.TestCase):
def test_data_put_filter_fbs(self):
endpoint = f"data/var"
url = f"{URL_BASE}{endpoint}"
url = f"{self.URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
filter = {"filter": {"var": {"index": [0, 1, 4]}}}
result = self.session.put(url, headers=header, json=filter)
@@ -254,7 +222,7 @@ class EndPoints(unittest.TestCase):
def test_data_put_single_var(self):
endpoint = f"data/var"
url = f"{URL_BASE}{endpoint}"
url = f"{self.URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
var_filter = {"filter": {"var": {"annotation_value": [{"name": index_col_name, "values": ["RER1"]}]}}}
@@ -268,6 +236,80 @@ class EndPoints(unittest.TestCase):
def test_static(self):
endpoint = "static"
file = "js/service-worker.js"
url = f"{LOCAL_URL}{endpoint}/{file}"
url = f"{self.LOCAL_URL}{endpoint}/{file}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
class EndPointsAnndata(unittest.TestCase, EndPoints):
"""Test Case for endpoints"""
PORT = 5010
LOCAL_URL = f"http://127.0.0.1:{PORT}/"
VERSION = "v0.2"
URL_BASE = f"{LOCAL_URL}api/{VERSION}/"
@classmethod
def setUpClass(cls):
cls.ps = Popen(
[
"cellxgene",
"--no-upgrade-check",
"launch",
"../example-dataset/pbmc3k.h5ad",
"--verbose",
"--port",
str(cls.PORT),
]
)
cls.session = requests.Session()
for i in range(90):
try:
result = cls.session.get(f"{cls.URL_BASE}schema")
cls.schema = result.json()
except requests.exceptions.ConnectionError:
time.sleep(1)
@classmethod
def tearDownClass(cls):
try:
cls.ps.terminate()
except ProcessLookupError:
pass
class EndPointsCxg(unittest.TestCase, EndPoints):
"""Test Case for endpoints"""
PORT = 5011
LOCAL_URL = f"http://127.0.0.1:{PORT}/"
VERSION = "v0.2"
URL_BASE = f"{LOCAL_URL}api/{VERSION}/"
@classmethod
def setUpClass(cls):
cls.ps = Popen(
[
"cellxgene",
"--no-upgrade-check",
"launch",
"../example-dataset/pbmc3k.cxg",
"--verbose",
"--port",
str(cls.PORT),
]
)
cls.session = requests.Session()
for i in range(90):
try:
result = cls.session.get(f"{cls.URL_BASE}schema")
cls.schema = result.json()
except requests.exceptions.ConnectionError:
time.sleep(1)
@classmethod
def tearDownClass(cls):
try:
cls.ps.terminate()
except ProcessLookupError:
pass
+1 -1
View File
@@ -4,7 +4,7 @@ import numpy as np
from scipy import sparse
import decode_fbs
from server.app.util.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
from server.data_common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
class FbsTests(unittest.TestCase):
@@ -5,9 +5,9 @@ import math
import decode_fbs
from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
from server.app.util.errors import FilterError
from server.app.util.data_locator import DataLocator
from server.data_anndata.anndata_adaptor import AnndataAdaptor
from server.common.errors import FilterError
from server.common.data_locator import DataLocator
class NaNTest(unittest.TestCase):
@@ -21,12 +21,12 @@ class NaNTest(unittest.TestCase):
}
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UserWarning)
self.data = ScanpyEngine(DataLocator("test/test_datasets/nan.h5ad"), self.args)
self.data = AnndataAdaptor(DataLocator("test/test_datasets/nan.h5ad"), self.args)
self.data._create_schema()
def test_load(self):
with self.assertWarns(UserWarning):
ScanpyEngine(DataLocator("test/test_datasets/nan.h5ad"), self.args)
AnndataAdaptor(DataLocator("test/test_datasets/nan.h5ad"), self.args)
def test_init(self):
self.assertEqual(self.data.cell_count, 100)
+22 -18
View File
@@ -8,9 +8,11 @@ import shutil
import numpy as np
import pandas as pd
from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
from server.app.util.fbs.matrix import encode_matrix_fbs
from server.app.util.data_locator import DataLocator
from server.data_anndata.anndata_adaptor import AnndataAdaptor
from server.data_common.fbs.matrix import encode_matrix_fbs
from server.common.data_locator import DataLocator
from server.common.annotations import AnnotationsLocalFile
from server.common.rest import schema_get_helper, annotations_put_fbs_helper
class WritableAnnotationTest(unittest.TestCase):
@@ -23,11 +25,11 @@ class WritableAnnotationTest(unittest.TestCase):
"obs_names": None,
"var_names": None,
"diffexp_lfc_cutoff": 0.01,
"annotations": True,
"annotations_file": self.annotations_file,
"annotations_output_dir": None,
}
self.data = ScanpyEngine(DataLocator("../example-dataset/pbmc3k.h5ad"), args)
fname = "../example-dataset/pbmc3k.h5ad"
data_locator = DataLocator(fname)
self.data = AnndataAdaptor(data_locator, args)
self.annotations = AnnotationsLocalFile(None, self.annotations_file)
def tearDown(self):
shutil.rmtree(self.tmpDir)
@@ -36,18 +38,19 @@ class WritableAnnotationTest(unittest.TestCase):
df = pd.DataFrame(data)
return encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
def annotation_put_fbs(self, fbs):
annotations_put_fbs_helper(self.data, self.annotations, fbs)
res = json.dumps({"status": "OK"})
return res
def test_error_checks(self):
# verify that the expected errors are generated
n_rows = self.data.data.obs.shape[0]
fbs_bad = self.make_fbs({"louvain": pd.Series(["undefined" for l in range(0, n_rows)], dtype="category")})
# ensure attempt to change VAR annotation
with self.assertRaises(ValueError):
self.data.annotation_put_fbs("var", fbs_bad)
# ensure we catch attempt to overwrite non-writable data
with self.assertRaises(KeyError):
self.data.annotation_put_fbs("obs", fbs_bad)
self.annotation_put_fbs(fbs_bad)
def test_write_to_file(self):
# verify the file is written as expected
@@ -58,7 +61,7 @@ class WritableAnnotationTest(unittest.TestCase):
"cat_B": pd.Series(["label_B" for l in range(0, n_rows)], dtype="category"),
}
)
res = self.data.annotation_put_fbs("obs", fbs)
res = self.annotation_put_fbs(fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
self.assertTrue(path.exists(self.annotations_file))
df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment="#")
@@ -75,7 +78,7 @@ class WritableAnnotationTest(unittest.TestCase):
"cat_C": pd.Series(["label_C" for l in range(0, n_rows)], dtype="category"),
}
)
res = self.data.annotation_put_fbs("obs", fbs)
res = self.annotation_put_fbs(fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
self.assertTrue(path.exists(self.annotations_file))
df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment="#")
@@ -100,7 +103,7 @@ class WritableAnnotationTest(unittest.TestCase):
}
)
for i in range(0, 11):
res = self.data.annotation_put_fbs("obs", fbs)
res = self.annotation_put_fbs(fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
name, ext = path.splitext(self.annotations_file)
@@ -122,12 +125,13 @@ class WritableAnnotationTest(unittest.TestCase):
)
# put
res = self.data.annotation_put_fbs("obs", fbs)
res = self.annotation_put_fbs(fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
# get
fbsAll = self.data.annotation_to_fbs_matrix("obs")
schema = self.data.get_schema()
labels = self.annotations.read_labels(None)
fbsAll = self.data.annotation_to_fbs_matrix("obs", None, labels)
schema = schema_get_helper(self.data, self.annotations)
annotations = decode_fbs.decode_matrix_FBS(fbsAll)
obs_index_col_name = schema["annotations"]["obs"]["index"]
self.assertEqual(annotations["n_rows"], n_rows)