From 58000c1815e44f15b7b8f56ef327ad7b23862144 Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Wed, 8 Aug 2018 14:42:45 -0700 Subject: [PATCH 1/5] Format loaded dataset - create cell and gene ids - recast numbers to float32/int32 --- server/app/scanpy_engine/scanpy_engine.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/server/app/scanpy_engine/scanpy_engine.py b/server/app/scanpy_engine/scanpy_engine.py index 6212f1d0..8148019d 100644 --- a/server/app/scanpy_engine/scanpy_engine.py +++ b/server/app/scanpy_engine/scanpy_engine.py @@ -13,6 +13,7 @@ class ScanpyEngine(CXGDriver): def __init__(self, data, schema=None, graph_method="umap", diffexp_method="ttest"): self.data = self._load_data(data) + self._format_data() self.schema = self._load_or_infer_schema(data, schema) self._set_cell_names() self.cell_count = self.data.shape[0] @@ -39,6 +40,7 @@ class ScanpyEngine(CXGDriver): def _load_data(data): return sc.read(os.path.join(data, "data.h5ad")) + # TODO delete after v0.1 v2.0 transition def _load_or_infer_schema(self, data, schema): if not os.path.isfile(os.path.join(data, schema)): # Initialize with cell name which is built off the index @@ -75,6 +77,23 @@ class ScanpyEngine(CXGDriver): data_schema = parse_schema(os.path.join(data, schema)) return data_schema + def _format_data(self): + # ensure gene + self.data.var["name"] = list(self.data.var.index) + self.data.var.index = list(range(self.data.var.shape[0])) + # ensure cell name + self.data.obs["name"] = list(self.data.obs.index) + self.data.obs.index = list(range(self.data.obs.shape[0])) + # ensure formats correct/ reform old format + for annotation in self.data.obs: + data_type = self.data.obs[annotation].dtype + if data_type.kind == 'f' and data_type != "float32": + self.data.obs[annotation] = self.data.obs[annotation].astype("float32") + elif data_type.kind in ['i', 'u'] and data_type != "int32": + self.data.obs[annotation] = self.data.obs[annotation].astype("int32") + if self.data.X.dtype != 'float32': + self.data.X = self.data.X.astype("float32") + def cells(self): return list(self.data.obs.index) From 55fa9b892a85e29fe3fa16b5a7603cc3c2a93b95 Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Fri, 10 Aug 2018 16:38:07 -0700 Subject: [PATCH 2/5] Splitting the format validation and mandatory annotations --- server/app/driver/driver.py | 4 -- server/app/scanpy_engine/scanpy_engine.py | 65 ++++------------------- 2 files changed, 11 insertions(+), 58 deletions(-) diff --git a/server/app/driver/driver.py b/server/app/driver/driver.py index fc6aae12..07f2e4a5 100644 --- a/server/app/driver/driver.py +++ b/server/app/driver/driver.py @@ -10,10 +10,6 @@ class CXGDriver(metaclass=ABCMeta): def _load_data(data): pass - @abstractmethod - def _load_or_infer_schema(data): - pass - @abstractmethod def cells(self): pass diff --git a/server/app/scanpy_engine/scanpy_engine.py b/server/app/scanpy_engine/scanpy_engine.py index 8148019d..89e8acb6 100644 --- a/server/app/scanpy_engine/scanpy_engine.py +++ b/server/app/scanpy_engine/scanpy_engine.py @@ -1,21 +1,20 @@ import os +import warnings import numpy as np +from pandas import Series import scanpy.api as sc from scipy import stats from server.app.app import cache from server.app.driver.driver import CXGDriver -from server.app.util.schema_parse import parse_schema - class ScanpyEngine(CXGDriver): - def __init__(self, data, schema=None, graph_method="umap", diffexp_method="ttest"): + def __init__(self, data, graph_method="umap", diffexp_method="ttest"): self.data = self._load_data(data) - self._format_data() - self.schema = self._load_or_infer_schema(data, schema) - self._set_cell_names() + self._validatate_data_types() + self._add_mandatory_annotations() self.cell_count = self.data.shape[0] self.gene_count = self.data.shape[1] self.graph_method = graph_method @@ -40,59 +39,17 @@ class ScanpyEngine(CXGDriver): def _load_data(data): return sc.read(os.path.join(data, "data.h5ad")) - # TODO delete after v0.1 v2.0 transition - def _load_or_infer_schema(self, data, schema): - if not os.path.isfile(os.path.join(data, schema)): - # Initialize with cell name which is built off the index - data_schema = { - "CellName": { - "type": "string", - "variabletype": "categorical", - "displayname": "Name", - "include": True - } - } - metadata_fields = list(self.data.obs) - for m in metadata_fields: - # Since there are many type of float/int in numpy datatypes the kind attribute of a datatype object - # offers a decent insight into whether it can be lumped in with floats or ints, which is what we - # care about here. - data_kind = self.data.obs[m].dtype.kind - variable_type = "categorical" - data_type = "string" - if data_kind == 'f': - variable_type = "continuous" - data_type = "float" - elif data_kind in ['i', 'u']: - data_type = "int" - if self.data.obs[m].nunique() > 50: - variable_type = "continuous" - data_schema[m] = { - "type": data_type, - "variabletype": variable_type, - "displayname": m, - "include": True - } - else: - data_schema = parse_schema(os.path.join(data, schema)) - return data_schema - - def _format_data(self): + def _add_mandatory_annotations(self): # ensure gene self.data.var["name"] = list(self.data.var.index) - self.data.var.index = list(range(self.data.var.shape[0])) + self.data.var.index = Series(list(range(self.data.var.shape[0])), dtype='int32') # ensure cell name self.data.obs["name"] = list(self.data.obs.index) - self.data.obs.index = list(range(self.data.obs.shape[0])) - # ensure formats correct/ reform old format - for annotation in self.data.obs: - data_type = self.data.obs[annotation].dtype - if data_type.kind == 'f' and data_type != "float32": - self.data.obs[annotation] = self.data.obs[annotation].astype("float32") - elif data_type.kind in ['i', 'u'] and data_type != "int32": - self.data.obs[annotation] = self.data.obs[annotation].astype("int32") + self.data.obs.index = Series(list(range(self.data.obs.shape[0])), dtype='int32') + + def _validatate_data_types(self): if self.data.X.dtype != 'float32': - self.data.X = self.data.X.astype("float32") + warnings.warn(f"Scanpy data matrix is in {self.data.X.dtype} format not float32. Precision may be truncated.") def cells(self): return list(self.data.obs.index) From d44d267bda540cd258b5ec3acbb5a4c2b58214c7 Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Fri, 10 Aug 2018 16:40:13 -0700 Subject: [PATCH 3/5] Initial tests for the rest v2 refactor testing the annotations and the data format validation --- .travis.yml | 3 -- pytest.ini | 3 ++ server/app/scanpy_engine/scanpy_engine.py | 4 +- server/test/test_scanpy_engine.py | 60 ++++------------------- 4 files changed, 15 insertions(+), 55 deletions(-) create mode 100644 pytest.ini diff --git a/.travis.yml b/.travis.yml index 0333cbda..1b509187 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,3 @@ script: - set -eo pipefail - flake8 server/app/ - pytest -s server/test/test_filter.py server/test/test_scanpy_engine.py - - cellxgene scanpy example-dataset/ & - - for i in {1..90}; do if http :5005/api/v0.1/initialize > /dev/null; then break; else echo "Waiting for server..."; sleep 1; fi; done - - pytest server/test/test_api.py diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..d776a2bd --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +filterwarnings = + ignore::UserWarning \ No newline at end of file diff --git a/server/app/scanpy_engine/scanpy_engine.py b/server/app/scanpy_engine/scanpy_engine.py index 89e8acb6..c08964be 100644 --- a/server/app/scanpy_engine/scanpy_engine.py +++ b/server/app/scanpy_engine/scanpy_engine.py @@ -9,6 +9,7 @@ from scipy import stats from server.app.app import cache from server.app.driver.driver import CXGDriver + class ScanpyEngine(CXGDriver): def __init__(self, data, graph_method="umap", diffexp_method="ttest"): @@ -49,7 +50,8 @@ class ScanpyEngine(CXGDriver): def _validatate_data_types(self): if self.data.X.dtype != 'float32': - warnings.warn(f"Scanpy data matrix is in {self.data.X.dtype} format not float32. Precision may be truncated.") + warnings.warn(f"Scanpy data matrix is in {self.data.X.dtype} format not float32. " + f"Precision may be truncated.") def cells(self): return list(self.data.obs.index) diff --git a/server/test/test_scanpy_engine.py b/server/test/test_scanpy_engine.py index e346fb6c..15bff582 100644 --- a/server/test/test_scanpy_engine.py +++ b/server/test/test_scanpy_engine.py @@ -5,7 +5,7 @@ from server.app.scanpy_engine.scanpy_engine import ScanpyEngine class UtilTest(unittest.TestCase): def setUp(self): - self.data = ScanpyEngine("example-dataset/", schema="data_schema.json") + self.data = ScanpyEngine("example-dataset/") def test_init(self): self.assertEqual(self.data.cell_count, 2638) @@ -13,57 +13,15 @@ class UtilTest(unittest.TestCase): epsilon = 0.000005 self.assertTrue(self.data.data.X[0,0] - -0.17146951 < epsilon) - def test_schema(self): - self.assertEqual(self.data.schema, {'CellName': {'type': 'string', 'variabletype': 'categorical', 'displayname': 'Name', 'include': True}, 'n_genes': {'type': 'int', 'variabletype': 'continuous', 'displayname': 'Num Genes', 'include': True}, 'percent_mito': {'type': 'float', 'variabletype': 'continuous', 'displayname': 'Mitochondrial Percentage', 'include': True}, 'n_counts': {'type': 'float', 'variabletype': 'continuous', 'displayname': 'Num Counts', 'include': True}, 'louvain': {'type': 'string', 'variabletype': 'categorical', 'displayname': 'Louvain Cluster', 'include': True}}) - - def test_cells(self): - cells = self.data.cells() - self.assertIn("AAACATACAACCAC-1", cells) - self.assertEqual(len(cells), 2638) - - def test_genes(self): - genes = self.data.genes() - self.assertIn("SEPT4", genes) - self.assertEqual(len(genes), 1838) - - def test_filter_categorical(self): - filter = {"louvain": {"variable_type": "categorical", "value_type": "string", "query": ["B cells"]}} - filtered_data = self.data.filter_cells(filter) - self.assertEqual(filtered_data.shape, (342, 1838)) - louvain_vals = filtered_data.obs['louvain'].tolist() - self.assertIn("B cells", louvain_vals) - self.assertNotIn("NK cells", louvain_vals) - - def test_filter_continuous(self): - # print(self.data.data.obs["n_genes"].tolist()) - filter = {"n_genes": {"variable_type": "continuous", "value_type": "int", "query": {"min": 300, "max": 400}}} - filtered_data = self.data.filter_cells(filter) - self.assertEqual(filtered_data.shape, (71, 1838)) - n_genes_vals = filtered_data.obs['n_genes'].tolist() - for val in n_genes_vals: - self.assertTrue(300 <= val <= 400) - - def test_metadata(self): - metadata = self.data.metadata(df=self.data.data) - self.assertEqual(len(metadata), 2638) - self.assertIn('louvain', metadata[0]) - - @unittest.skip("Umap not producing the same graph on different systems, even with the same seed. Skipping for now") - def test_create_graph(self): - graph = self.data.create_graph(df=self.data.data) - self.assertEqual(graph[0][1], 0.5545382653143183) - self.assertEqual(graph[0][2], 0.6021833809031731) - - def test_diffexp(self): - diffexp = self.data.diffexp(["AAACATACAACCAC-1", "AACCGATGGTCATG-1"], ["CCGATAGACCTAAG-1", "GGTGGAGAAGTAGA-1"], 0.5, 7) - self.assertEqual(diffexp["celllist1"]["topgenes"], ['EBNA1BP2', 'DIAPH1', 'SLC25A11', 'SNRNP27', 'COMMD8', 'COTL1', 'GTF3A']) - - def test_expression(self): - expression = self.data.expression(cells=["AAACATACAACCAC-1"]) - data_exp = self.data.data[["AAACATACAACCAC-1"], :].X - for idx in range(len(expression["cells"][0]["e"])): - self.assertEqual(expression["cells"][0]["e"][idx], data_exp[idx]) + def test_mandatory_annotations(self): + self.assertIn("name", self.data.data.obs) + self.assertEqual(list(self.data.data.obs.index), list(range(2638))) + self.assertIn("name", self.data.data.var) + self.assertEqual(list(self.data.data.var.index), list(range(1838))) + def test_data_type(self): + self.data.data.X = self.data.data.X.astype("float64") + self.assertWarns(UserWarning, self.data._validatate_data_types()) if __name__ == '__main__': unittest.main() From a29ba8da4e90da2cadf25ee1d0866603084ce8e4 Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Mon, 13 Aug 2018 15:00:06 -0700 Subject: [PATCH 4/5] Limit pytest ignore warning to only one specific warning --- pytest.ini | 3 --- server/test/test_api.py | 1 - server/test/test_scanpy_engine.py | 3 +++ 3 files changed, 3 insertions(+), 4 deletions(-) delete mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index d776a2bd..00000000 --- a/pytest.ini +++ /dev/null @@ -1,3 +0,0 @@ -[pytest] -filterwarnings = - ignore::UserWarning \ No newline at end of file diff --git a/server/test/test_api.py b/server/test/test_api.py index b004740a..a87f79fc 100644 --- a/server/test/test_api.py +++ b/server/test/test_api.py @@ -51,4 +51,3 @@ class EndPoints(unittest.TestCase): url = "{base}{endpoint}".format(base=self.url_base, endpoint="diffexpression") result = self.session.post(url, data=json.dumps({"celllist1": ["AAACATACAACCAC-1", "AACCGATGGTCATG-1"], "celllist2": ["CCGATAGACCTAAG-1", "GGTGGAGAAGTAGA-1"]}), headers={'content-type': 'application/json'}) assert result.status_code == 200 - \ No newline at end of file diff --git a/server/test/test_scanpy_engine.py b/server/test/test_scanpy_engine.py index 15bff582..124f6e3b 100644 --- a/server/test/test_scanpy_engine.py +++ b/server/test/test_scanpy_engine.py @@ -1,4 +1,5 @@ import unittest +import pytest from server.app.scanpy_engine.scanpy_engine import ScanpyEngine @@ -19,9 +20,11 @@ class UtilTest(unittest.TestCase): self.assertIn("name", self.data.data.var) self.assertEqual(list(self.data.data.var.index), list(range(1838))) + @pytest.mark.filterwarnings("ignore:Scanpy data matrix") def test_data_type(self): self.data.data.X = self.data.data.X.astype("float64") self.assertWarns(UserWarning, self.data._validatate_data_types()) + if __name__ == '__main__': unittest.main() From ddaa7016a599778f3e843cefd9d0d2f55b17d205 Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Mon, 13 Aug 2018 15:00:23 -0700 Subject: [PATCH 5/5] Quote formatting ' => " --- server/app/scanpy_engine/scanpy_engine.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/app/scanpy_engine/scanpy_engine.py b/server/app/scanpy_engine/scanpy_engine.py index c08964be..0e9918e0 100644 --- a/server/app/scanpy_engine/scanpy_engine.py +++ b/server/app/scanpy_engine/scanpy_engine.py @@ -43,13 +43,13 @@ class ScanpyEngine(CXGDriver): def _add_mandatory_annotations(self): # ensure gene self.data.var["name"] = list(self.data.var.index) - self.data.var.index = Series(list(range(self.data.var.shape[0])), dtype='int32') + self.data.var.index = Series(list(range(self.data.var.shape[0])), dtype="int32") # ensure cell name self.data.obs["name"] = list(self.data.obs.index) - self.data.obs.index = Series(list(range(self.data.obs.shape[0])), dtype='int32') + self.data.obs.index = Series(list(range(self.data.obs.shape[0])), dtype="int32") def _validatate_data_types(self): - if self.data.X.dtype != 'float32': + if self.data.X.dtype != "float32": warnings.warn(f"Scanpy data matrix is in {self.data.X.dtype} format not float32. " f"Precision may be truncated.")