From 187bbfdcf7e6cc60f74fbba2ad1be870808c4845 Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Wed, 3 Oct 2018 15:09:46 -0700 Subject: [PATCH] /data/var (#295) * Upgrade version of scanpy * /data/var This works for everything except the case where there is only one gene. Anndata flattens X when there is only one var thus causing the transpose to fail. * Fix edge case when an axis (obs/var) only contains 1 element --- server/app/rest_api/rest.py | 101 ++++++++++++++++++- server/app/scanpy_engine/scanpy_engine.py | 29 ++++-- server/requirements.txt | 2 +- server/test/test_api.py | 116 ++++++++++++---------- server/test/test_scanpy_engine.py | 42 ++++++-- 5 files changed, 221 insertions(+), 69 deletions(-) diff --git a/server/app/rest_api/rest.py b/server/app/rest_api/rest.py index 2eea3a77..1d3f7c48 100644 --- a/server/app/rest_api/rest.py +++ b/server/app/rest_api/rest.py @@ -478,7 +478,7 @@ class DataObsAPI(Resource): return make_response(e.message, HTTPStatus.BAD_REQUEST) df = current_app.data.filter_dataframe(filter_, include_uns=False) if accept_type and accept_type[0] == "application/json": - return make_response((jsonify(current_app.data.data_frame(df)))) + return make_response((jsonify(current_app.data.data_frame(df, axis=Axis.OBS)))) # TODO support CSV else: return make_response(f"Unsupported accept-type: {accept_type}", HTTPStatus.NOT_ACCEPTABLE) @@ -520,7 +520,103 @@ class DataObsAPI(Resource): # TODO catch error for bad filter df = current_app.data.filter_dataframe(request.get_json()["filter"], include_uns=False) if request.accept_mimetypes.best_match(['application/json']): - return make_response((jsonify(current_app.data.data_frame(df)))) + return make_response((jsonify(current_app.data.data_frame(df, axis=Axis.OBS)))) + # TODO support CSV + else: + return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) + + +class DataVarAPI(Resource): + @swagger.doc({ + "summary": "Get data (expression values) from the dataframe.", + "tags": ["data"], + "parameters": [ + { + "in": "query", + "name": "filter", + "type": "string", + "description": "axis:key:value" + }, + { + "in": "query", + "name": "accept-type", + "type": "string", + "description": "MIME type" + }, + ], + "responses": { + "200": { + "description": "expression", + "examples": { + "application/json": { + "obs": [0, 20000], + "var": [ + [1, 39483, 3902, 203, 0, 0, 28] + ] + } + } + }, + "400": { + "description": "Malformed filter" + }, + "406": { + "description": "Unacceptable MIME type" + }, + } + }) + def get(self): + # request.args is immutable + args = dict(request.args) + accept_type = args.pop("accept-type", None) + try: + filter_ = parse_filter(ImmutableMultiDict(args), current_app.data.schema['annotations']) + except QueryStringError as e: + return make_response(e.message, HTTPStatus.BAD_REQUEST) + df = current_app.data.filter_dataframe(filter_, include_uns=False) + if accept_type and accept_type[0] == "application/json": + return make_response((jsonify(current_app.data.data_frame(df, axis=Axis.VAR)))) + # TODO support CSV + else: + return make_response(f"Unsupported accept-type: {accept_type}", HTTPStatus.NOT_ACCEPTABLE) + + @swagger.doc({ + "summary": "Get data (expression values) from the dataframe.", + "tags": ["data"], + "parameters": [ + { + 'name': 'filter', + 'description': 'Complex Filter', + 'in': 'body', + 'schema': FilterModel + } + ], + "responses": { + "200": { + "description": "expression", + "examples": { + "application/json": { + "obs": [0, 20000], + "var": [ + [1, 39483, 3902, 203, 0, 0, 28] + ] + } + } + }, + "400": { + "description": "Malformed filter" + }, + "406": { + "description": "Unacceptable MIME type" + }, + } + }) + def put(self): + if not request.accept_mimetypes.best_match(["application/json", "text/csv"]): + return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) + # TODO catch error for bad filter + df = current_app.data.filter_dataframe(request.get_json()["filter"], include_uns=False) + if request.accept_mimetypes.best_match(['application/json']): + return make_response((jsonify(current_app.data.data_frame(df, axis=Axis.VAR)))) # TODO support CSV else: return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) @@ -536,4 +632,5 @@ def get_api_resources(): api.add_resource(DiffExpObsAPI, "/diffexp/obs") api.add_resource(AnnotationsVarAPI, "/annotations/var") api.add_resource(DataObsAPI, "/data/obs") + api.add_resource(DataVarAPI, "/data/var") return api diff --git a/server/app/scanpy_engine/scanpy_engine.py b/server/app/scanpy_engine/scanpy_engine.py index 5c795261..38c8f829 100644 --- a/server/app/scanpy_engine/scanpy_engine.py +++ b/server/app/scanpy_engine/scanpy_engine.py @@ -280,7 +280,7 @@ class ScanpyEngine(CXGDriver): return sorted(result, key=lambda gene: gene[0]) # @cache.memoize() - def data_frame(self, df): + def data_frame(self, df, axis): """ Retrieves data for each variable for observations in data frame :param df: from filter_cells, dataframe @@ -289,9 +289,24 @@ class ScanpyEngine(CXGDriver): "obs": [cellid, var1 expression, var2 expression, ...], } """ - var_index = df.var.index.tolist() - expression = DataFrame(df.X, index=df.obs.index) - return { - "var": var_index, - "obs": expression.reset_index().values.tolist() - } + var_idx = df.var.index.tolist() + obs_idx = df.obs.index.tolist() + values = df.X + df_shape = df.shape + if df_shape[0] == 1: + values = values[None, :] + elif df_shape[1] == 1: + values = values[:, None] + if axis == Axis.OBS: + expression = DataFrame(values, index=obs_idx) + result = { + "var": var_idx, + "obs": expression.reset_index().values.tolist() + } + else: + expression = DataFrame(values.T, index=var_idx) + result = { + "obs": obs_idx, + "var": expression.reset_index().values.tolist(), + } + return result diff --git a/server/requirements.txt b/server/requirements.txt index 152c4e1a..6d7fa098 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -7,5 +7,5 @@ Flask-RESTful==0.3.6 flask-restful-swagger-2==0.35 numpy==1.14.5 pandas==0.23.1 -scanpy==1.0.4 +scanpy==1.3.1 scipy==1.1.0 diff --git a/server/test/test_api.py b/server/test/test_api.py index ed329030..10a3ff9c 100644 --- a/server/test/test_api.py +++ b/server/test/test_api.py @@ -265,71 +265,81 @@ class EndPoints(unittest.TestCase): self.assertEqual(len(result_data["data"]), 2) def test_get_data(self): - endpoint = "data/obs" - query = "accept-type=application/json" - url = f"{URL_BASE}{endpoint}?{query}" - result = self.session.get(url) - self.assertEqual(result.status_code, 200) - result_data = result.json() - self.assertEqual(len(result_data["obs"]), 2638) + for axis in ["obs", "var"]: + endpoint = f"data/{axis}" + query = "accept-type=application/json" + url = f"{URL_BASE}{endpoint}?{query}" + result = self.session.get(url) + self.assertEqual(result.status_code, 200) + result_data = result.json() + self.assertEqual(len(result_data["obs"]), 2638) def test_data_mimetype_error(self): - endpoint = "data/obs" - query = "accept-type=xxx" - url = f"{URL_BASE}{endpoint}?{query}" - result = self.session.get(url) - self.assertEqual(result.status_code, 406) - # no accept type - url = f"{URL_BASE}{endpoint}" - result = self.session.get(url) - self.assertEqual(result.status_code, 406) + for axis in ["obs", "var"]: + endpoint = f"data/{axis}" + query = "accept-type=xxx" + url = f"{URL_BASE}{endpoint}?{query}" + result = self.session.get(url) + self.assertEqual(result.status_code, 406) + # no accept type + url = f"{URL_BASE}{endpoint}" + result = self.session.get(url) + self.assertEqual(result.status_code, 406) def test_data_filter(self): - endpoint = "data/obs" - query = "accept-type=application/json&obs:louvain=NK cells&obs:louvain=CD8 T cells&obs:n_counts=3000,*" - url = f"{URL_BASE}{endpoint}?{query}" - result = self.session.get(url) - self.assertEqual(result.status_code, 200) - result_data = result.json() - self.assertEqual(len(result_data["obs"]), 38) + for axis in ["obs", "var"]: + endpoint = f"data/{axis}" + query = "accept-type=application/json&obs:louvain=NK cells&obs:louvain=CD8 T cells&obs:n_counts=3000,*" + url = f"{URL_BASE}{endpoint}?{query}" + result = self.session.get(url) + self.assertEqual(result.status_code, 200) + result_data = result.json() + self.assertEqual(len(result_data["obs"]), 38) def test_data_put(self): - endpoint = "data/obs" - url = f"{URL_BASE}{endpoint}" - header = {"Accept": "application/json"} - obs_filter = { - "filter": { - "obs": { - "annotation_value": [ - {"name": "louvain", "values": ["NK cells", "CD8 T cells"]}, - {"name": "n_counts", "min": 3000}, - ], - "index": [1, 99, [1000, 2000]] + for axis in ["obs", "var"]: + endpoint = f"data/{axis}" + url = f"{URL_BASE}{endpoint}" + header = {"Accept": "application/json"} + obs_filter = { + "filter": { + "obs": { + "annotation_value": [ + {"name": "louvain", "values": ["NK cells", "CD8 T cells"]}, + {"name": "n_counts", "min": 3000}, + ], + "index": [1, 99, [1000, 2000]] + } } } - } - result = self.session.put(url, headers=header, json=obs_filter) - self.assertEqual(result.status_code, 200) - result_data = result.json() - self.assertEqual(len(result_data["obs"]), 15) + result = self.session.put(url, headers=header, json=obs_filter) + self.assertEqual(result.status_code, 200) + result_data = result.json() + self.assertEqual(len(result_data["obs"]), 15) def test_data_put_single_var(self): - endpoint = "data/obs" - url = f"{URL_BASE}{endpoint}" - header = {"Accept": "application/json"} - var_filter = { - "filter": { - "var": { - "annotation_value": [ - {"name": "name", "values": ["RER1"]}, - ] + for axis in ["obs", "var"]: + endpoint = f"data/{axis}" + url = f"{URL_BASE}{endpoint}" + header = {"Accept": "application/json"} + var_filter = { + "filter": { + "var": { + "annotation_value": [ + {"name": "name", "values": ["RER1"]}, + ] + } } } - } - result = self.session.put(url, headers=header, json=var_filter) - self.assertEqual(result.status_code, 200) - result_data = result.json() - self.assertEqual(len(result_data["obs"][0]), 2) + result = self.session.put(url, headers=header, json=var_filter) + self.assertEqual(result.status_code, 200) + result_data = result.json() + if axis == "obs": + self.assertEqual(len(result_data["obs"][0]), 2) + self.assertEqual(len(result_data["var"]), 1) + elif axis == "var": + self.assertEqual(len(result_data["obs"]), 2638) + self.assertEqual(len(result_data["var"][0]), 2639) def test_static(self): endpoint = "static" diff --git a/server/test/test_scanpy_engine.py b/server/test/test_scanpy_engine.py index 1dc5ee30..f0b15292 100644 --- a/server/test/test_scanpy_engine.py +++ b/server/test/test_scanpy_engine.py @@ -206,9 +206,12 @@ class UtilTest(unittest.TestCase): self.assertEqual(len(result), 20) def test_data_frame(self): - data_frame = self.data.data_frame(self.data.data) - self.assertEqual(len(data_frame["var"]), 1838) - self.assertEqual(len(data_frame["obs"]), 2638) + data_frame_obs = self.data.data_frame(self.data.data, "obs") + self.assertEqual(len(data_frame_obs["var"]), 1838) + self.assertEqual(len(data_frame_obs["obs"]), 2638) + data_frame_var = self.data.data_frame(self.data.data, "var") + self.assertEqual(len(data_frame_var["var"]), 1838) + self.assertEqual(len(data_frame_var["obs"]), 2638) def test_filtered_data_frame(self): filter_ = { @@ -221,9 +224,36 @@ class UtilTest(unittest.TestCase): } } data = self.data.filter_dataframe(filter_["filter"]) - data_frame = self.data.data_frame(data) - self.assertEqual(len(data_frame["var"]), 1838) - self.assertEqual(len(data_frame["obs"]), 497) + data_frame_obs = self.data.data_frame(data, "obs") + self.assertEqual(len(data_frame_obs["var"]), 1838) + self.assertEqual(len(data_frame_obs["obs"]), 497) + self.assertEqual(type(data_frame_obs["obs"][0]), list) + self.assertEqual(type(data_frame_obs["var"][0]), int) + data_frame_var = self.data.data_frame(data, "var") + self.assertEqual(len(data_frame_var["var"]), 1838) + self.assertEqual(len(data_frame_var["obs"]), 497) + self.assertEqual(type(data_frame_var["var"][0]), list) + self.assertEqual(type(data_frame_var["obs"][0]), int) + + def test_data_single_gene(self): + for axis in ["obs", "var"]: + filter_ = { + "filter": { + "var": { + "annotation_value": [ + {"name": "name", "values": ["RER1"]}, + ] + } + } + } + data = self.data.filter_dataframe(filter_["filter"], include_uns=False) + data_frame_var = self.data.data_frame(data, axis) + if axis == "obs": + self.assertEqual(type(data_frame_var["var"][0]), int) + self.assertEqual(type(data_frame_var["obs"][0]), list) + elif axis == "var": + self.assertEqual(type(data_frame_var["obs"][0]), int) + self.assertEqual(type(data_frame_var["var"][0]), list) if __name__ == '__main__': unittest.main()