From b6f253e8432ac5ad8015d4439afbd7a5eb2ad5dd Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Thu, 5 Jul 2018 17:06:51 -0700 Subject: [PATCH 1/5] Initial packaging working pip install works if client files are built and moved manually --- MANIFEST.in | 1 + server/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index ec6f0e6c..b881e298 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ recursive-include server/app/web/templates * recursive-include server/app/web/static * + diff --git a/server/requirements.txt b/server/requirements.txt index 21b90d84..7ab96abb 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,5 +1,5 @@ aniso8601==3.0.2 -anndata==0.6.4 +anndata==0.6.1 certifi==2018.4.16 chardet==3.0.4 click==6.7 From b8f501a46c4633c111900ac6a297d7763d9cc59b Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Wed, 11 Jul 2018 18:01:39 -0700 Subject: [PATCH 2/5] Move to running tests after installing to avoid the sys.path.insert --- server/test/test_filter.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/server/test/test_filter.py b/server/test/test_filter.py index f46cd14a..3641fee1 100644 --- a/server/test/test_filter.py +++ b/server/test/test_filter.py @@ -1,9 +1,7 @@ import unittest - from unittest.mock import MagicMock -import sys -sys.path.insert(0, "../app") -from util.filter import _convert_variable, parse_filter + +from server.app.util.filter import _convert_variable, parse_filter class UtilTest(unittest.TestCase): @@ -72,3 +70,6 @@ class UtilTest(unittest.TestCase): filterMock.getlist.return_value = ["0,*"] query = parse_filter(filterMock, self.schema) assert query == {"n_genes": {"variable_type": "continuous", "value_type": "int", "query": {"min": 0, "max": None}}} + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From 05b9a6833d378af8cad88d4babb72247b723c3ae Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Wed, 11 Jul 2018 18:02:28 -0700 Subject: [PATCH 3/5] Add tests for scanpy engine --- server/app/scanpy_engine/scanpy_engine.py | 6 +-- server/test/test_scanpy_engine.py | 49 +++++++++++++++++++++++ 2 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 server/test/test_scanpy_engine.py diff --git a/server/app/scanpy_engine/scanpy_engine.py b/server/app/scanpy_engine/scanpy_engine.py index 142f87a7..a3eac916 100644 --- a/server/app/scanpy_engine/scanpy_engine.py +++ b/server/app/scanpy_engine/scanpy_engine.py @@ -61,11 +61,11 @@ class ScanpyEngine(CXGDriver): else: min_ = value["query"]["min"] max_ = value["query"]["max"] - if min_: + if min_ is not None: key_idx = np.array((getattr(self.data.obs, key) >= min_).data) cell_idx = np.logical_and(cell_idx, key_idx) - if max_: - key_idx = np.array((getattr(self.data.obs, key) <= min_).data) + if max_ is not None: + key_idx = np.array((getattr(self.data.obs, key) <= max_).data) cell_idx = np.logical_and(cell_idx, key_idx) return self.data[cell_idx, :] diff --git a/server/test/test_scanpy_engine.py b/server/test/test_scanpy_engine.py new file mode 100644 index 00000000..2ecfdf7f --- /dev/null +++ b/server/test/test_scanpy_engine.py @@ -0,0 +1,49 @@ +import unittest + +from server.app.scanpy_engine.scanpy_engine import ScanpyEngine + + +class UtilTest(unittest.TestCase): + def setUp(self): + self.data = ScanpyEngine("example-dataset/", schema="data_schema.json") + + def test_init(self): + assert self.data.cell_count == 2638 + assert self.data.gene_count == 1838 + epsilon = 0.000005 + assert self.data.data.X[0,0] - -0.17146951 < epsilon + + def test_schema(self): + assert self.data.schema == {'CellName': {'type': 'string', 'variabletype': 'categorical', 'displayname': 'Name', 'include': True}, 'n_genes': {'type': 'int', 'variabletype': 'continuous', 'displayname': 'Num Genes', 'include': True}, 'percent_mito': {'type': 'float', 'variabletype': 'continuous', 'displayname': 'Mitochondrial Percentage', 'include': True}, 'n_counts': {'type': 'float', 'variabletype': 'continuous', 'displayname': 'Num Counts', 'include': True}, 'louvain': {'type': 'string', 'variabletype': 'categorical', 'displayname': 'Louvain Cluster', 'include': True}} + + def test_cells(self): + cells = self.data.cells() + assert "AAACATACAACCAC-1" in cells + assert len(cells) == 2638 + + def test_genes(self): + genes = self.data.genes() + assert "SEPT4" in genes + assert len(genes) == 1838 + + def test_filter_categorical(self): + filter = {"louvain": {"variable_type": "categorical", "value_type": "string", "query": ["B cells"]}} + filtered_data = self.data.filter_cells(filter) + assert filtered_data.shape == (342, 1838) + louvain_vals = filtered_data.obs['louvain'].tolist() + assert "B cells" in louvain_vals + assert "NK cells" not in louvain_vals + + def test_filter_continuous(self): + # print(self.data.data.obs["n_genes"].tolist()) + filter = {"n_genes": {"variable_type": "continuous", "value_type": "int", "query": {"min": 300, "max": 400}}} + filtered_data = self.data.filter_cells(filter) + assert filtered_data.shape == (71, 1838) + n_genes_vals = filtered_data.obs['n_genes'].tolist() + for val in n_genes_vals: + assert 300 <= val <= 400 + + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From a5423a69117cb2f06a5825c8a10a059115d57bee Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Fri, 13 Jul 2018 15:22:05 -0700 Subject: [PATCH 4/5] Adding minor tests to scanpy calculation code --- server/app/scanpy_engine/scanpy_engine.py | 1 + server/test/test_scanpy_engine.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/server/app/scanpy_engine/scanpy_engine.py b/server/app/scanpy_engine/scanpy_engine.py index a3eac916..4e580efc 100644 --- a/server/app/scanpy_engine/scanpy_engine.py +++ b/server/app/scanpy_engine/scanpy_engine.py @@ -124,6 +124,7 @@ class ScanpyEngine(CXGDriver): expression_1 = self.data.X[cells_idx_1, :] expression_2 = self.data.X[cells_idx_2, :] diff_exp = stats.ttest_ind(expression_1, expression_2) + # TODO break this up into functions set1 = np.logical_and(diff_exp.pvalue < pval, diff_exp.statistic > 0) set2 = np.logical_and(diff_exp.pvalue < pval, diff_exp.statistic < 0) stat1 = diff_exp.statistic[set1] diff --git a/server/test/test_scanpy_engine.py b/server/test/test_scanpy_engine.py index 2ecfdf7f..1e7d91d5 100644 --- a/server/test/test_scanpy_engine.py +++ b/server/test/test_scanpy_engine.py @@ -43,6 +43,25 @@ class UtilTest(unittest.TestCase): for val in n_genes_vals: assert 300 <= val <= 400 + def test_metadata(self): + metadata = self.data.metadata(df=self.data.data) + assert len(metadata) == 2638 + assert 'louvain' in metadata[0] + + def test_create_graph(self): + graph = self.data.create_graph(df=self.data.data) + assert graph[0][1] == 0.5545382653143183 + assert graph[0][2] == 0.6021833809031731 + + def test_diffexp(self): + diffexp = self.data.diffexp(["AAACATACAACCAC-1", "AACCGATGGTCATG-1"], ["CCGATAGACCTAAG-1", "GGTGGAGAAGTAGA-1"], 0.5, 7) + assert diffexp["celllist1"]["topgenes"] == ['EBNA1BP2', 'DIAPH1', 'SLC25A11', 'SNRNP27', 'COMMD8', 'COTL1', 'GTF3A'] + + def test_expression(self): + expression = self.data.expression(cells=["AAACATACAACCAC-1"]) + data_exp = self.data.data[["AAACATACAACCAC-1"], :].X + for idx in range(len(expression["cells"][0]["e"])): + assert expression["cells"][0]["e"][idx] == data_exp[idx] if __name__ == '__main__': From fdaaaf69a241e724db54815402bb1c5c956035c7 Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Tue, 17 Jul 2018 11:01:38 -0700 Subject: [PATCH 5/5] use unittest assert methods instead of assert --- server/test/test_scanpy_engine.py | 40 +++++++++++++++---------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/server/test/test_scanpy_engine.py b/server/test/test_scanpy_engine.py index 1e7d91d5..e99198ca 100644 --- a/server/test/test_scanpy_engine.py +++ b/server/test/test_scanpy_engine.py @@ -8,61 +8,61 @@ class UtilTest(unittest.TestCase): self.data = ScanpyEngine("example-dataset/", schema="data_schema.json") def test_init(self): - assert self.data.cell_count == 2638 - assert self.data.gene_count == 1838 + self.assertEqual(self.data.cell_count, 2638) + self.assertEqual(self.data.gene_count, 1838) epsilon = 0.000005 - assert self.data.data.X[0,0] - -0.17146951 < epsilon + self.assertTrue(self.data.data.X[0,0] - -0.17146951 < epsilon) def test_schema(self): - assert self.data.schema == {'CellName': {'type': 'string', 'variabletype': 'categorical', 'displayname': 'Name', 'include': True}, 'n_genes': {'type': 'int', 'variabletype': 'continuous', 'displayname': 'Num Genes', 'include': True}, 'percent_mito': {'type': 'float', 'variabletype': 'continuous', 'displayname': 'Mitochondrial Percentage', 'include': True}, 'n_counts': {'type': 'float', 'variabletype': 'continuous', 'displayname': 'Num Counts', 'include': True}, 'louvain': {'type': 'string', 'variabletype': 'categorical', 'displayname': 'Louvain Cluster', 'include': True}} + self.assertEqual(self.data.schema, {'CellName': {'type': 'string', 'variabletype': 'categorical', 'displayname': 'Name', 'include': True}, 'n_genes': {'type': 'int', 'variabletype': 'continuous', 'displayname': 'Num Genes', 'include': True}, 'percent_mito': {'type': 'float', 'variabletype': 'continuous', 'displayname': 'Mitochondrial Percentage', 'include': True}, 'n_counts': {'type': 'float', 'variabletype': 'continuous', 'displayname': 'Num Counts', 'include': True}, 'louvain': {'type': 'string', 'variabletype': 'categorical', 'displayname': 'Louvain Cluster', 'include': True}}) def test_cells(self): cells = self.data.cells() - assert "AAACATACAACCAC-1" in cells - assert len(cells) == 2638 + self.assertIn("AAACATACAACCAC-1", cells) + self.assertEqual(len(cells), 2638) def test_genes(self): genes = self.data.genes() - assert "SEPT4" in genes - assert len(genes) == 1838 + self.assertIn("SEPT4", genes) + self.assertEqual(len(genes), 1838) def test_filter_categorical(self): filter = {"louvain": {"variable_type": "categorical", "value_type": "string", "query": ["B cells"]}} filtered_data = self.data.filter_cells(filter) - assert filtered_data.shape == (342, 1838) + self.assertEqual(filtered_data.shape, (342, 1838)) louvain_vals = filtered_data.obs['louvain'].tolist() - assert "B cells" in louvain_vals - assert "NK cells" not in louvain_vals + self.assertIn("B cells", louvain_vals) + self.assertNotIn("NK cells", louvain_vals) def test_filter_continuous(self): # print(self.data.data.obs["n_genes"].tolist()) filter = {"n_genes": {"variable_type": "continuous", "value_type": "int", "query": {"min": 300, "max": 400}}} filtered_data = self.data.filter_cells(filter) - assert filtered_data.shape == (71, 1838) + self.assertEqual(filtered_data.shape, (71, 1838)) n_genes_vals = filtered_data.obs['n_genes'].tolist() for val in n_genes_vals: - assert 300 <= val <= 400 + self.assertTrue(300 <= val <= 400) def test_metadata(self): metadata = self.data.metadata(df=self.data.data) - assert len(metadata) == 2638 - assert 'louvain' in metadata[0] + self.assertEqual(len(metadata), 2638) + self.assertIn('louvain', metadata[0]) def test_create_graph(self): graph = self.data.create_graph(df=self.data.data) - assert graph[0][1] == 0.5545382653143183 - assert graph[0][2] == 0.6021833809031731 + self.assertEqual(graph[0][1], 0.5545382653143183) + self.assertEqual(graph[0][2], 0.6021833809031731) def test_diffexp(self): diffexp = self.data.diffexp(["AAACATACAACCAC-1", "AACCGATGGTCATG-1"], ["CCGATAGACCTAAG-1", "GGTGGAGAAGTAGA-1"], 0.5, 7) - assert diffexp["celllist1"]["topgenes"] == ['EBNA1BP2', 'DIAPH1', 'SLC25A11', 'SNRNP27', 'COMMD8', 'COTL1', 'GTF3A'] + self.assertEqual(diffexp["celllist1"]["topgenes"], ['EBNA1BP2', 'DIAPH1', 'SLC25A11', 'SNRNP27', 'COMMD8', 'COTL1', 'GTF3A']) def test_expression(self): expression = self.data.expression(cells=["AAACATACAACCAC-1"]) data_exp = self.data.data[["AAACATACAACCAC-1"], :].X for idx in range(len(expression["cells"][0]["e"])): - assert expression["cells"][0]["e"][idx] == data_exp[idx] + self.assertEqual(expression["cells"][0]["e"][idx], data_exp[idx]) if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main()