mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-09-30 19:28:12 +08:00
server refactor (#1140)
This PR contains a refactoring to make adding new features easier. The new features include supporting the tiledb format, and the multi dataset application. The refactoring includes Simplifying the directory structure and files. a class structure to handle annotations (currently one type: AnnotationsLocalFile). a class to handle application configuration a class structure to handle matrix data (currently AnndataAdaptor and CxgAdaptor). CxgAdaptor uses tiledb. Algorithms that were previously dependent on the scanpy anndata object are now generalized to work with an abstract interface. The multi dataset option is not fully supported yet, and so the option to use it is hidden. Use "cli launch --dataroot ..." To access this feature. All combinations of app single dataset/ app multi dataset and AnndataAdaptor/CxgAdaptor work with all the features, such as annotations, ontologies, diffexp.
This commit is contained in:
@@ -2,17 +2,17 @@
|
||||
Code to decode, for testing purposes, the flatbuffer encoded blobs.
|
||||
This code will need to be updated if fbs/matrix.fbs changes.
|
||||
|
||||
For more information, see fbs/matrix.fbs and server/app/util/fbs/
|
||||
For more information, see fbs/matrix.fbs and server/data_common/fbs/
|
||||
"""
|
||||
import json
|
||||
|
||||
import server.app.util.fbs.NetEncoding.TypedArray as TypedArray
|
||||
import server.app.util.fbs.NetEncoding.Matrix as Matrix
|
||||
import server.app.util.fbs.NetEncoding.Int32Array as Int32Array
|
||||
import server.app.util.fbs.NetEncoding.Uint32Array as Uint32Array
|
||||
import server.app.util.fbs.NetEncoding.Float32Array as Float32Array
|
||||
import server.app.util.fbs.NetEncoding.Float64Array as Float64Array
|
||||
import server.app.util.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
|
||||
import server.data_common.fbs.NetEncoding.TypedArray as TypedArray
|
||||
import server.data_common.fbs.NetEncoding.Matrix as Matrix
|
||||
import server.data_common.fbs.NetEncoding.Int32Array as Int32Array
|
||||
import server.data_common.fbs.NetEncoding.Uint32Array as Uint32Array
|
||||
import server.data_common.fbs.NetEncoding.Float32Array as Float32Array
|
||||
import server.data_common.fbs.NetEncoding.Float64Array as Float64Array
|
||||
import server.data_common.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
|
||||
|
||||
|
||||
def decode_typed_array(tarr):
|
||||
|
||||
@@ -9,12 +9,12 @@ from parameterized import parameterized_class
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
|
||||
from server.app.util.errors import FilterError, DisabledFeatureError
|
||||
from server.app.util.data_locator import DataLocator
|
||||
from server.data_anndata.anndata_adaptor import AnndataAdaptor
|
||||
from server.common.errors import FilterError
|
||||
from server.common.data_locator import DataLocator
|
||||
|
||||
"""
|
||||
Test the scanpy engine using the pbmc3k data set.
|
||||
Test the anndata adaptor using the pbmc3k data set.
|
||||
"""
|
||||
|
||||
|
||||
@@ -29,7 +29,7 @@ Test the scanpy engine using the pbmc3k data set.
|
||||
("test/test_datasets/pbmc3k-CSR-gz.h5ad", True),
|
||||
],
|
||||
)
|
||||
class EngineTest(unittest.TestCase):
|
||||
class AdaptorTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
args = {
|
||||
"layout": ["umap"],
|
||||
@@ -37,10 +37,9 @@ class EngineTest(unittest.TestCase):
|
||||
"obs_names": None,
|
||||
"var_names": None,
|
||||
"diffexp_lfc_cutoff": 0.01,
|
||||
"layout_file": None,
|
||||
"backed": self.backed,
|
||||
"anndata_backed": self.backed,
|
||||
}
|
||||
self.data = ScanpyEngine(DataLocator(self.data_locator), args)
|
||||
self.data = AnndataAdaptor(DataLocator(self.data_locator), args)
|
||||
|
||||
def test_init(self):
|
||||
self.assertEqual(self.data.cell_count, 2638)
|
||||
@@ -56,7 +55,7 @@ class EngineTest(unittest.TestCase):
|
||||
self.assertIn(var_index_col_name, self.data.data.var)
|
||||
self.assertEqual(list(self.data.data.var.index), list(range(1838)))
|
||||
|
||||
@pytest.mark.filterwarnings("ignore:Scanpy data matrix")
|
||||
@pytest.mark.filterwarnings("ignore:Anndata data matrix")
|
||||
def test_data_type(self):
|
||||
# don't run the test on the more exotic data types, as they don't
|
||||
# support the astype() interface (used by this test, but not underlying app)
|
||||
@@ -98,9 +97,7 @@ class EngineTest(unittest.TestCase):
|
||||
self.data._create_schema()
|
||||
|
||||
def test_config(self):
|
||||
self.assertEqual(
|
||||
self.data.features["layout"]["obs"], {"available": True, "interactiveLimit": 50000},
|
||||
)
|
||||
self.assertEqual(self.data.get_features()["layout_obs"].available, True)
|
||||
|
||||
def test_layout(self):
|
||||
fbs = self.data.layout_to_fbs_matrix()
|
||||
@@ -142,10 +139,6 @@ class EngineTest(unittest.TestCase):
|
||||
self.assertEqual(annotations["n_rows"], 1838)
|
||||
self.assertEqual(annotations["n_cols"], 1)
|
||||
|
||||
def test_annotation_put(self):
|
||||
with self.assertRaises(DisabledFeatureError):
|
||||
self.data.annotation_put_fbs(None, "obs")
|
||||
|
||||
def test_diffexp_topN(self):
|
||||
f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
|
||||
f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
|
||||
+8
-38
@@ -1,49 +1,20 @@
|
||||
import unittest
|
||||
import json
|
||||
|
||||
from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
|
||||
from server.app.util.errors import DriverError
|
||||
from server.app.util.data_locator import DataLocator
|
||||
from server.data_anndata.anndata_adaptor import AnndataAdaptor
|
||||
from server.common.data_locator import DataLocator
|
||||
|
||||
|
||||
class DataLoadEngineTest(unittest.TestCase):
|
||||
class DataLoadAdaptorTest(unittest.TestCase):
|
||||
"""
|
||||
Test file loading, including deferred loading/update.
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
self.data_file = DataLocator("../example-dataset/pbmc3k.h5ad")
|
||||
self.data = ScanpyEngine()
|
||||
|
||||
def test_init(self):
|
||||
self.assertIsNone(self.data.data)
|
||||
|
||||
def test_delayed_load_args(self):
|
||||
args = {
|
||||
"layout": ["tsne"],
|
||||
"max_category_items": 1000,
|
||||
"obs_names": "foo",
|
||||
"var_names": "bar",
|
||||
"diffexp_lfc_cutoff": 0.1,
|
||||
"annotations": False,
|
||||
"annotations_file": None,
|
||||
"annotations_output_dir": None,
|
||||
"backed": False,
|
||||
"diffexp_may_be_slow": False,
|
||||
"disable_diffexp": False,
|
||||
"annotations_cell_ontology_enabled": False,
|
||||
"annotations_cell_ontology_obopath": None,
|
||||
"annotations_cell_ontology_terms": None,
|
||||
}
|
||||
self.data.update(args=args)
|
||||
self.assertEqual(args, self.data.config)
|
||||
|
||||
def test_requires_data(self):
|
||||
with self.assertRaises(DriverError):
|
||||
self.data._create_schema()
|
||||
self.data = AnndataAdaptor(self.data_file)
|
||||
|
||||
def test_delayed_load_data(self):
|
||||
self.data.update(data_locator=self.data_file)
|
||||
self.data._create_schema()
|
||||
self.assertEqual(self.data.cell_count, 2638)
|
||||
self.assertEqual(self.data.gene_count, 1838)
|
||||
@@ -51,7 +22,6 @@ class DataLoadEngineTest(unittest.TestCase):
|
||||
self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
|
||||
|
||||
def test_diffexp_topN(self):
|
||||
self.data.update(data_locator=self.data_file)
|
||||
f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
|
||||
f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
|
||||
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
|
||||
@@ -60,7 +30,7 @@ class DataLoadEngineTest(unittest.TestCase):
|
||||
self.assertEqual(len(result), 20)
|
||||
|
||||
|
||||
class DataLocatorEngineTest(unittest.TestCase):
|
||||
class DataLocatorAdaptorTest(unittest.TestCase):
|
||||
"""
|
||||
Test various types of data locators we expect to consume
|
||||
"""
|
||||
@@ -82,17 +52,17 @@ class DataLocatorEngineTest(unittest.TestCase):
|
||||
|
||||
def test_posix_file(self):
|
||||
locator = DataLocator("../example-dataset/pbmc3k.h5ad")
|
||||
data = ScanpyEngine(locator, self.args)
|
||||
data = AnndataAdaptor(locator, self.args)
|
||||
self.stdAsserts(data)
|
||||
|
||||
def test_url_https(self):
|
||||
url = "https://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/example-dataset/pbmc3k.h5ad"
|
||||
locator = DataLocator(url)
|
||||
data = ScanpyEngine(locator, self.args)
|
||||
data = AnndataAdaptor(locator, self.args)
|
||||
self.stdAsserts(data)
|
||||
|
||||
def test_url_http(self):
|
||||
url = "http://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/example-dataset/pbmc3k.h5ad"
|
||||
locator = DataLocator(url)
|
||||
data = ScanpyEngine(locator, self.args)
|
||||
data = AnndataAdaptor(locator, self.args)
|
||||
self.stdAsserts(data)
|
||||
+97
-55
@@ -7,50 +7,18 @@ import requests
|
||||
|
||||
import decode_fbs
|
||||
|
||||
LOCAL_URL = "http://127.0.0.1:5005/"
|
||||
VERSION = "v0.2"
|
||||
URL_BASE = f"{LOCAL_URL}api/{VERSION}/"
|
||||
|
||||
BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
|
||||
|
||||
|
||||
class EndPoints(unittest.TestCase):
|
||||
"""Test Case for endpoints"""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.ps = Popen(
|
||||
[
|
||||
"cellxgene",
|
||||
"--no-upgrade-check",
|
||||
"launch",
|
||||
"../example-dataset/pbmc3k.h5ad",
|
||||
"--verbose",
|
||||
"--port",
|
||||
"5005",
|
||||
]
|
||||
)
|
||||
session = requests.Session()
|
||||
for i in range(90):
|
||||
try:
|
||||
result = session.get(f"{URL_BASE}schema")
|
||||
cls.schema = result.json()
|
||||
except requests.exceptions.ConnectionError:
|
||||
time.sleep(1)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
try:
|
||||
cls.ps.terminate()
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
class EndPoints(object):
|
||||
|
||||
def setUp(self):
|
||||
self.session = requests.Session()
|
||||
|
||||
def test_initialize(self):
|
||||
endpoint = "schema"
|
||||
url = f"{URL_BASE}{endpoint}"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
result = self.session.get(url)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
@@ -61,18 +29,18 @@ class EndPoints(unittest.TestCase):
|
||||
|
||||
def test_config(self):
|
||||
endpoint = "config"
|
||||
url = f"{URL_BASE}{endpoint}"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
result = self.session.get(url)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
result_data = result.json()
|
||||
self.assertIn("library_versions", result_data["config"])
|
||||
self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k")
|
||||
self.assertEqual(len(result_data["config"]["features"]), 4)
|
||||
self.assertEqual(len(result_data["config"]["features"]), 3)
|
||||
|
||||
def test_get_layout_fbs(self):
|
||||
endpoint = "layout/obs"
|
||||
url = f"{URL_BASE}{endpoint}"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
header = {"Accept": "application/octet-stream"}
|
||||
result = self.session.get(url, headers=header)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
@@ -81,22 +49,22 @@ class EndPoints(unittest.TestCase):
|
||||
self.assertEqual(df["n_rows"], 2638)
|
||||
self.assertEqual(df["n_cols"], 8)
|
||||
self.assertIsNotNone(df["columns"])
|
||||
self.assertListEqual(
|
||||
df["col_idx"],
|
||||
["pca_0", "pca_1", "tsne_0", "tsne_1", "umap_0", "umap_1", "draw_graph_fr_0", "draw_graph_fr_1"],
|
||||
self.assertSetEqual(
|
||||
set(df["col_idx"]),
|
||||
set(["pca_0", "pca_1", "tsne_0", "tsne_1", "umap_0", "umap_1", "draw_graph_fr_0", "draw_graph_fr_1"]),
|
||||
)
|
||||
self.assertIsNone(df["row_idx"])
|
||||
self.assertEqual(len(df["columns"]), df["n_cols"])
|
||||
|
||||
def test_bad_filter(self):
|
||||
endpoint = "data/var"
|
||||
url = f"{URL_BASE}{endpoint}"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
result = self.session.put(url, json=BAD_FILTER)
|
||||
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
|
||||
|
||||
def test_get_annotations_obs_fbs(self):
|
||||
endpoint = "annotations/obs"
|
||||
url = f"{URL_BASE}{endpoint}"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
header = {"Accept": "application/octet-stream"}
|
||||
result = self.session.get(url, headers=header)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
@@ -114,7 +82,7 @@ class EndPoints(unittest.TestCase):
|
||||
def test_get_annotations_obs_keys_fbs(self):
|
||||
endpoint = "annotations/obs"
|
||||
query = "annotation-name=n_genes&annotation-name=percent_mito"
|
||||
url = f"{URL_BASE}{endpoint}?{query}"
|
||||
url = f"{self.URL_BASE}{endpoint}?{query}"
|
||||
header = {"Accept": "application/octet-stream"}
|
||||
result = self.session.get(url, headers=header)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
@@ -131,13 +99,13 @@ class EndPoints(unittest.TestCase):
|
||||
def test_get_annotations_obs_error(self):
|
||||
endpoint = "annotations/obs"
|
||||
query = "annotation-name=notakey"
|
||||
url = f"{URL_BASE}{endpoint}?{query}"
|
||||
url = f"{self.URL_BASE}{endpoint}?{query}"
|
||||
result = self.session.get(url)
|
||||
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
|
||||
|
||||
def test_diff_exp(self):
|
||||
endpoint = "diffexp/obs"
|
||||
url = f"{URL_BASE}{endpoint}"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
params = {
|
||||
"mode": "topN",
|
||||
"set1": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["NK cells"]}]}}},
|
||||
@@ -152,7 +120,7 @@ class EndPoints(unittest.TestCase):
|
||||
|
||||
def test_diff_exp_indices(self):
|
||||
endpoint = "diffexp/obs"
|
||||
url = f"{URL_BASE}{endpoint}"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
params = {
|
||||
"mode": "topN",
|
||||
"count": 10,
|
||||
@@ -167,7 +135,7 @@ class EndPoints(unittest.TestCase):
|
||||
|
||||
def test_get_annotations_var_fbs(self):
|
||||
endpoint = "annotations/var"
|
||||
url = f"{URL_BASE}{endpoint}"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
header = {"Accept": "application/octet-stream"}
|
||||
result = self.session.get(url, headers=header)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
@@ -185,7 +153,7 @@ class EndPoints(unittest.TestCase):
|
||||
def test_get_annotations_var_keys_fbs(self):
|
||||
endpoint = "annotations/var"
|
||||
query = "annotation-name=n_cells"
|
||||
url = f"{URL_BASE}{endpoint}?{query}"
|
||||
url = f"{self.URL_BASE}{endpoint}?{query}"
|
||||
header = {"Accept": "application/octet-stream"}
|
||||
result = self.session.get(url, headers=header)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
@@ -202,27 +170,27 @@ class EndPoints(unittest.TestCase):
|
||||
def test_get_annotations_var_error(self):
|
||||
endpoint = "annotations/var"
|
||||
query = "annotation-name=notakey"
|
||||
url = f"{URL_BASE}{endpoint}?{query}"
|
||||
url = f"{self.URL_BASE}{endpoint}?{query}"
|
||||
result = self.session.get(url)
|
||||
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
|
||||
|
||||
def test_data_mimetype_error(self):
|
||||
endpoint = f"data/var"
|
||||
header = {"Accept": "xxx"}
|
||||
url = f"{URL_BASE}{endpoint}"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
result = self.session.put(url, headers=header)
|
||||
self.assertEqual(result.status_code, HTTPStatus.NOT_ACCEPTABLE)
|
||||
|
||||
def test_fbs_default(self):
|
||||
endpoint = f"data/var"
|
||||
url = f"{URL_BASE}{endpoint}"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
result = self.session.put(url)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
|
||||
|
||||
def test_data_put_fbs(self):
|
||||
endpoint = f"data/var"
|
||||
url = f"{URL_BASE}{endpoint}"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
header = {"Accept": "application/octet-stream"}
|
||||
result = self.session.put(url, headers=header)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
@@ -237,7 +205,7 @@ class EndPoints(unittest.TestCase):
|
||||
|
||||
def test_data_put_filter_fbs(self):
|
||||
endpoint = f"data/var"
|
||||
url = f"{URL_BASE}{endpoint}"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
header = {"Accept": "application/octet-stream"}
|
||||
filter = {"filter": {"var": {"index": [0, 1, 4]}}}
|
||||
result = self.session.put(url, headers=header, json=filter)
|
||||
@@ -254,7 +222,7 @@ class EndPoints(unittest.TestCase):
|
||||
|
||||
def test_data_put_single_var(self):
|
||||
endpoint = f"data/var"
|
||||
url = f"{URL_BASE}{endpoint}"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
header = {"Accept": "application/octet-stream"}
|
||||
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
|
||||
var_filter = {"filter": {"var": {"annotation_value": [{"name": index_col_name, "values": ["RER1"]}]}}}
|
||||
@@ -268,6 +236,80 @@ class EndPoints(unittest.TestCase):
|
||||
def test_static(self):
|
||||
endpoint = "static"
|
||||
file = "js/service-worker.js"
|
||||
url = f"{LOCAL_URL}{endpoint}/{file}"
|
||||
url = f"{self.LOCAL_URL}{endpoint}/{file}"
|
||||
result = self.session.get(url)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
|
||||
|
||||
class EndPointsAnndata(unittest.TestCase, EndPoints):
|
||||
"""Test Case for endpoints"""
|
||||
|
||||
PORT = 5010
|
||||
LOCAL_URL = f"http://127.0.0.1:{PORT}/"
|
||||
VERSION = "v0.2"
|
||||
URL_BASE = f"{LOCAL_URL}api/{VERSION}/"
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.ps = Popen(
|
||||
[
|
||||
"cellxgene",
|
||||
"--no-upgrade-check",
|
||||
"launch",
|
||||
"../example-dataset/pbmc3k.h5ad",
|
||||
"--verbose",
|
||||
"--port",
|
||||
str(cls.PORT),
|
||||
]
|
||||
)
|
||||
cls.session = requests.Session()
|
||||
for i in range(90):
|
||||
try:
|
||||
result = cls.session.get(f"{cls.URL_BASE}schema")
|
||||
cls.schema = result.json()
|
||||
except requests.exceptions.ConnectionError:
|
||||
time.sleep(1)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
try:
|
||||
cls.ps.terminate()
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
|
||||
class EndPointsCxg(unittest.TestCase, EndPoints):
|
||||
"""Test Case for endpoints"""
|
||||
|
||||
PORT = 5011
|
||||
LOCAL_URL = f"http://127.0.0.1:{PORT}/"
|
||||
VERSION = "v0.2"
|
||||
URL_BASE = f"{LOCAL_URL}api/{VERSION}/"
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.ps = Popen(
|
||||
[
|
||||
"cellxgene",
|
||||
"--no-upgrade-check",
|
||||
"launch",
|
||||
"../example-dataset/pbmc3k.cxg",
|
||||
"--verbose",
|
||||
"--port",
|
||||
str(cls.PORT),
|
||||
]
|
||||
)
|
||||
cls.session = requests.Session()
|
||||
for i in range(90):
|
||||
try:
|
||||
result = cls.session.get(f"{cls.URL_BASE}schema")
|
||||
cls.schema = result.json()
|
||||
except requests.exceptions.ConnectionError:
|
||||
time.sleep(1)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
try:
|
||||
cls.ps.terminate()
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
@@ -4,7 +4,7 @@ import numpy as np
|
||||
from scipy import sparse
|
||||
|
||||
import decode_fbs
|
||||
from server.app.util.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
|
||||
from server.data_common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
|
||||
|
||||
|
||||
class FbsTests(unittest.TestCase):
|
||||
|
||||
@@ -5,9 +5,9 @@ import math
|
||||
|
||||
import decode_fbs
|
||||
|
||||
from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
|
||||
from server.app.util.errors import FilterError
|
||||
from server.app.util.data_locator import DataLocator
|
||||
from server.data_anndata.anndata_adaptor import AnndataAdaptor
|
||||
from server.common.errors import FilterError
|
||||
from server.common.data_locator import DataLocator
|
||||
|
||||
|
||||
class NaNTest(unittest.TestCase):
|
||||
@@ -21,12 +21,12 @@ class NaNTest(unittest.TestCase):
|
||||
}
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", category=UserWarning)
|
||||
self.data = ScanpyEngine(DataLocator("test/test_datasets/nan.h5ad"), self.args)
|
||||
self.data = AnndataAdaptor(DataLocator("test/test_datasets/nan.h5ad"), self.args)
|
||||
self.data._create_schema()
|
||||
|
||||
def test_load(self):
|
||||
with self.assertWarns(UserWarning):
|
||||
ScanpyEngine(DataLocator("test/test_datasets/nan.h5ad"), self.args)
|
||||
AnndataAdaptor(DataLocator("test/test_datasets/nan.h5ad"), self.args)
|
||||
|
||||
def test_init(self):
|
||||
self.assertEqual(self.data.cell_count, 100)
|
||||
@@ -8,9 +8,11 @@ import shutil
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
|
||||
from server.app.util.fbs.matrix import encode_matrix_fbs
|
||||
from server.app.util.data_locator import DataLocator
|
||||
from server.data_anndata.anndata_adaptor import AnndataAdaptor
|
||||
from server.data_common.fbs.matrix import encode_matrix_fbs
|
||||
from server.common.data_locator import DataLocator
|
||||
from server.common.annotations import AnnotationsLocalFile
|
||||
from server.common.rest import schema_get_helper, annotations_put_fbs_helper
|
||||
|
||||
|
||||
class WritableAnnotationTest(unittest.TestCase):
|
||||
@@ -23,11 +25,11 @@ class WritableAnnotationTest(unittest.TestCase):
|
||||
"obs_names": None,
|
||||
"var_names": None,
|
||||
"diffexp_lfc_cutoff": 0.01,
|
||||
"annotations": True,
|
||||
"annotations_file": self.annotations_file,
|
||||
"annotations_output_dir": None,
|
||||
}
|
||||
self.data = ScanpyEngine(DataLocator("../example-dataset/pbmc3k.h5ad"), args)
|
||||
fname = "../example-dataset/pbmc3k.h5ad"
|
||||
data_locator = DataLocator(fname)
|
||||
self.data = AnndataAdaptor(data_locator, args)
|
||||
self.annotations = AnnotationsLocalFile(None, self.annotations_file)
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.tmpDir)
|
||||
@@ -36,18 +38,19 @@ class WritableAnnotationTest(unittest.TestCase):
|
||||
df = pd.DataFrame(data)
|
||||
return encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
|
||||
|
||||
def annotation_put_fbs(self, fbs):
|
||||
annotations_put_fbs_helper(self.data, self.annotations, fbs)
|
||||
res = json.dumps({"status": "OK"})
|
||||
return res
|
||||
|
||||
def test_error_checks(self):
|
||||
# verify that the expected errors are generated
|
||||
n_rows = self.data.data.obs.shape[0]
|
||||
fbs_bad = self.make_fbs({"louvain": pd.Series(["undefined" for l in range(0, n_rows)], dtype="category")})
|
||||
|
||||
# ensure attempt to change VAR annotation
|
||||
with self.assertRaises(ValueError):
|
||||
self.data.annotation_put_fbs("var", fbs_bad)
|
||||
|
||||
# ensure we catch attempt to overwrite non-writable data
|
||||
with self.assertRaises(KeyError):
|
||||
self.data.annotation_put_fbs("obs", fbs_bad)
|
||||
self.annotation_put_fbs(fbs_bad)
|
||||
|
||||
def test_write_to_file(self):
|
||||
# verify the file is written as expected
|
||||
@@ -58,7 +61,7 @@ class WritableAnnotationTest(unittest.TestCase):
|
||||
"cat_B": pd.Series(["label_B" for l in range(0, n_rows)], dtype="category"),
|
||||
}
|
||||
)
|
||||
res = self.data.annotation_put_fbs("obs", fbs)
|
||||
res = self.annotation_put_fbs(fbs)
|
||||
self.assertEqual(res, json.dumps({"status": "OK"}))
|
||||
self.assertTrue(path.exists(self.annotations_file))
|
||||
df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment="#")
|
||||
@@ -75,7 +78,7 @@ class WritableAnnotationTest(unittest.TestCase):
|
||||
"cat_C": pd.Series(["label_C" for l in range(0, n_rows)], dtype="category"),
|
||||
}
|
||||
)
|
||||
res = self.data.annotation_put_fbs("obs", fbs)
|
||||
res = self.annotation_put_fbs(fbs)
|
||||
self.assertEqual(res, json.dumps({"status": "OK"}))
|
||||
self.assertTrue(path.exists(self.annotations_file))
|
||||
df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment="#")
|
||||
@@ -100,7 +103,7 @@ class WritableAnnotationTest(unittest.TestCase):
|
||||
}
|
||||
)
|
||||
for i in range(0, 11):
|
||||
res = self.data.annotation_put_fbs("obs", fbs)
|
||||
res = self.annotation_put_fbs(fbs)
|
||||
self.assertEqual(res, json.dumps({"status": "OK"}))
|
||||
|
||||
name, ext = path.splitext(self.annotations_file)
|
||||
@@ -122,12 +125,13 @@ class WritableAnnotationTest(unittest.TestCase):
|
||||
)
|
||||
|
||||
# put
|
||||
res = self.data.annotation_put_fbs("obs", fbs)
|
||||
res = self.annotation_put_fbs(fbs)
|
||||
self.assertEqual(res, json.dumps({"status": "OK"}))
|
||||
|
||||
# get
|
||||
fbsAll = self.data.annotation_to_fbs_matrix("obs")
|
||||
schema = self.data.get_schema()
|
||||
labels = self.annotations.read_labels(None)
|
||||
fbsAll = self.data.annotation_to_fbs_matrix("obs", None, labels)
|
||||
schema = schema_get_helper(self.data, self.annotations)
|
||||
annotations = decode_fbs.decode_matrix_FBS(fbsAll)
|
||||
obs_index_col_name = schema["annotations"]["obs"]["index"]
|
||||
self.assertEqual(annotations["n_rows"], n_rows)
|
||||
|
||||
Reference in New Issue
Block a user