diff --git a/.travis.yml b/.travis.yml index 1b509187..0333cbda 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,3 +14,6 @@ script: - set -eo pipefail - flake8 server/app/ - pytest -s server/test/test_filter.py server/test/test_scanpy_engine.py + - cellxgene scanpy example-dataset/ & + - for i in {1..90}; do if http :5005/api/v0.1/initialize > /dev/null; then break; else echo "Waiting for server..."; sleep 1; fi; done + - pytest server/test/test_api.py diff --git a/server/app/driver/driver.py b/server/app/driver/driver.py index 07f2e4a5..fc6aae12 100644 --- a/server/app/driver/driver.py +++ b/server/app/driver/driver.py @@ -10,6 +10,10 @@ class CXGDriver(metaclass=ABCMeta): def _load_data(data): pass + @abstractmethod + def _load_or_infer_schema(data): + pass + @abstractmethod def cells(self): pass diff --git a/server/app/scanpy_engine/scanpy_engine.py b/server/app/scanpy_engine/scanpy_engine.py index 0e9918e0..6212f1d0 100644 --- a/server/app/scanpy_engine/scanpy_engine.py +++ b/server/app/scanpy_engine/scanpy_engine.py @@ -1,21 +1,20 @@ import os -import warnings import numpy as np -from pandas import Series import scanpy.api as sc from scipy import stats from server.app.app import cache from server.app.driver.driver import CXGDriver +from server.app.util.schema_parse import parse_schema class ScanpyEngine(CXGDriver): - def __init__(self, data, graph_method="umap", diffexp_method="ttest"): + def __init__(self, data, schema=None, graph_method="umap", diffexp_method="ttest"): self.data = self._load_data(data) - self._validatate_data_types() - self._add_mandatory_annotations() + self.schema = self._load_or_infer_schema(data, schema) + self._set_cell_names() self.cell_count = self.data.shape[0] self.gene_count = self.data.shape[1] self.graph_method = graph_method @@ -40,18 +39,41 @@ class ScanpyEngine(CXGDriver): def _load_data(data): return sc.read(os.path.join(data, "data.h5ad")) - def _add_mandatory_annotations(self): - # ensure gene - self.data.var["name"] = list(self.data.var.index) - self.data.var.index = Series(list(range(self.data.var.shape[0])), dtype="int32") - # ensure cell name - self.data.obs["name"] = list(self.data.obs.index) - self.data.obs.index = Series(list(range(self.data.obs.shape[0])), dtype="int32") - - def _validatate_data_types(self): - if self.data.X.dtype != "float32": - warnings.warn(f"Scanpy data matrix is in {self.data.X.dtype} format not float32. " - f"Precision may be truncated.") + def _load_or_infer_schema(self, data, schema): + if not os.path.isfile(os.path.join(data, schema)): + # Initialize with cell name which is built off the index + data_schema = { + "CellName": { + "type": "string", + "variabletype": "categorical", + "displayname": "Name", + "include": True + } + } + metadata_fields = list(self.data.obs) + for m in metadata_fields: + # Since there are many type of float/int in numpy datatypes the kind attribute of a datatype object + # offers a decent insight into whether it can be lumped in with floats or ints, which is what we + # care about here. + data_kind = self.data.obs[m].dtype.kind + variable_type = "categorical" + data_type = "string" + if data_kind == 'f': + variable_type = "continuous" + data_type = "float" + elif data_kind in ['i', 'u']: + data_type = "int" + if self.data.obs[m].nunique() > 50: + variable_type = "continuous" + data_schema[m] = { + "type": data_type, + "variabletype": variable_type, + "displayname": m, + "include": True + } + else: + data_schema = parse_schema(os.path.join(data, schema)) + return data_schema def cells(self): return list(self.data.obs.index) diff --git a/server/test/test_scanpy_engine.py b/server/test/test_scanpy_engine.py index 124f6e3b..e346fb6c 100644 --- a/server/test/test_scanpy_engine.py +++ b/server/test/test_scanpy_engine.py @@ -1,12 +1,11 @@ import unittest -import pytest from server.app.scanpy_engine.scanpy_engine import ScanpyEngine class UtilTest(unittest.TestCase): def setUp(self): - self.data = ScanpyEngine("example-dataset/") + self.data = ScanpyEngine("example-dataset/", schema="data_schema.json") def test_init(self): self.assertEqual(self.data.cell_count, 2638) @@ -14,16 +13,56 @@ class UtilTest(unittest.TestCase): epsilon = 0.000005 self.assertTrue(self.data.data.X[0,0] - -0.17146951 < epsilon) - def test_mandatory_annotations(self): - self.assertIn("name", self.data.data.obs) - self.assertEqual(list(self.data.data.obs.index), list(range(2638))) - self.assertIn("name", self.data.data.var) - self.assertEqual(list(self.data.data.var.index), list(range(1838))) + def test_schema(self): + self.assertEqual(self.data.schema, {'CellName': {'type': 'string', 'variabletype': 'categorical', 'displayname': 'Name', 'include': True}, 'n_genes': {'type': 'int', 'variabletype': 'continuous', 'displayname': 'Num Genes', 'include': True}, 'percent_mito': {'type': 'float', 'variabletype': 'continuous', 'displayname': 'Mitochondrial Percentage', 'include': True}, 'n_counts': {'type': 'float', 'variabletype': 'continuous', 'displayname': 'Num Counts', 'include': True}, 'louvain': {'type': 'string', 'variabletype': 'categorical', 'displayname': 'Louvain Cluster', 'include': True}}) - @pytest.mark.filterwarnings("ignore:Scanpy data matrix") - def test_data_type(self): - self.data.data.X = self.data.data.X.astype("float64") - self.assertWarns(UserWarning, self.data._validatate_data_types()) + def test_cells(self): + cells = self.data.cells() + self.assertIn("AAACATACAACCAC-1", cells) + self.assertEqual(len(cells), 2638) + + def test_genes(self): + genes = self.data.genes() + self.assertIn("SEPT4", genes) + self.assertEqual(len(genes), 1838) + + def test_filter_categorical(self): + filter = {"louvain": {"variable_type": "categorical", "value_type": "string", "query": ["B cells"]}} + filtered_data = self.data.filter_cells(filter) + self.assertEqual(filtered_data.shape, (342, 1838)) + louvain_vals = filtered_data.obs['louvain'].tolist() + self.assertIn("B cells", louvain_vals) + self.assertNotIn("NK cells", louvain_vals) + + def test_filter_continuous(self): + # print(self.data.data.obs["n_genes"].tolist()) + filter = {"n_genes": {"variable_type": "continuous", "value_type": "int", "query": {"min": 300, "max": 400}}} + filtered_data = self.data.filter_cells(filter) + self.assertEqual(filtered_data.shape, (71, 1838)) + n_genes_vals = filtered_data.obs['n_genes'].tolist() + for val in n_genes_vals: + self.assertTrue(300 <= val <= 400) + + def test_metadata(self): + metadata = self.data.metadata(df=self.data.data) + self.assertEqual(len(metadata), 2638) + self.assertIn('louvain', metadata[0]) + + @unittest.skip("Umap not producing the same graph on different systems, even with the same seed. Skipping for now") + def test_create_graph(self): + graph = self.data.create_graph(df=self.data.data) + self.assertEqual(graph[0][1], 0.5545382653143183) + self.assertEqual(graph[0][2], 0.6021833809031731) + + def test_diffexp(self): + diffexp = self.data.diffexp(["AAACATACAACCAC-1", "AACCGATGGTCATG-1"], ["CCGATAGACCTAAG-1", "GGTGGAGAAGTAGA-1"], 0.5, 7) + self.assertEqual(diffexp["celllist1"]["topgenes"], ['EBNA1BP2', 'DIAPH1', 'SLC25A11', 'SNRNP27', 'COMMD8', 'COTL1', 'GTF3A']) + + def test_expression(self): + expression = self.data.expression(cells=["AAACATACAACCAC-1"]) + data_exp = self.data.data[["AAACATACAACCAC-1"], :].X + for idx in range(len(expression["cells"][0]["e"])): + self.assertEqual(expression["cells"][0]["e"][idx], data_exp[idx]) if __name__ == '__main__':