/data/var (#295)

* Upgrade version of scanpy

* /data/var

This works for everything except the case where there is only one gene. Anndata flattens X when there is only one var thus causing the transpose to fail.

* Fix edge case when an axis (obs/var) only contains 1 element

server/app/rest_api/rest.py

@@ -478,7 +478,7 @@ class DataObsAPI(Resource):
             return make_response(e.message, HTTPStatus.BAD_REQUEST)
         df = current_app.data.filter_dataframe(filter_, include_uns=False)
         if accept_type and accept_type[0] == "application/json":
-            return make_response((jsonify(current_app.data.data_frame(df))))
+            return make_response((jsonify(current_app.data.data_frame(df, axis=Axis.OBS))))
         # TODO support CSV
         else:
             return make_response(f"Unsupported accept-type: {accept_type}", HTTPStatus.NOT_ACCEPTABLE)
@@ -520,7 +520,103 @@ class DataObsAPI(Resource):
         # TODO catch error for bad filter
         df = current_app.data.filter_dataframe(request.get_json()["filter"], include_uns=False)
         if request.accept_mimetypes.best_match(['application/json']):
-            return make_response((jsonify(current_app.data.data_frame(df))))
+            return make_response((jsonify(current_app.data.data_frame(df, axis=Axis.OBS))))
+        # TODO support CSV
+        else:
+            return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
+
+
+class DataVarAPI(Resource):
+    @swagger.doc({
+        "summary": "Get data (expression values) from the dataframe.",
+        "tags": ["data"],
+        "parameters": [
+            {
+                "in": "query",
+                "name": "filter",
+                "type": "string",
+                "description": "axis:key:value"
+            },
+            {
+                "in": "query",
+                "name": "accept-type",
+                "type": "string",
+                "description": "MIME type"
+            },
+        ],
+        "responses": {
+            "200": {
+                "description": "expression",
+                "examples": {
+                    "application/json": {
+                        "obs": [0, 20000],
+                        "var": [
+                            [1, 39483, 3902, 203, 0, 0, 28]
+                        ]
+                    }
+                }
+            },
+            "400": {
+                "description": "Malformed filter"
+            },
+            "406": {
+                "description": "Unacceptable MIME type"
+            },
+        }
+    })
+    def get(self):
+        # request.args is immutable
+        args = dict(request.args)
+        accept_type = args.pop("accept-type", None)
+        try:
+            filter_ = parse_filter(ImmutableMultiDict(args), current_app.data.schema['annotations'])
+        except QueryStringError as e:
+            return make_response(e.message, HTTPStatus.BAD_REQUEST)
+        df = current_app.data.filter_dataframe(filter_, include_uns=False)
+        if accept_type and accept_type[0] == "application/json":
+            return make_response((jsonify(current_app.data.data_frame(df, axis=Axis.VAR))))
+        # TODO support CSV
+        else:
+            return make_response(f"Unsupported accept-type: {accept_type}", HTTPStatus.NOT_ACCEPTABLE)
+
+    @swagger.doc({
+        "summary": "Get data (expression values) from the dataframe.",
+        "tags": ["data"],
+        "parameters": [
+            {
+                'name': 'filter',
+                'description': 'Complex Filter',
+                'in': 'body',
+                'schema': FilterModel
+            }
+        ],
+        "responses": {
+            "200": {
+                "description": "expression",
+                "examples": {
+                    "application/json": {
+                        "obs": [0, 20000],
+                        "var": [
+                            [1, 39483, 3902, 203, 0, 0, 28]
+                        ]
+                    }
+                }
+            },
+            "400": {
+                "description": "Malformed filter"
+            },
+            "406": {
+                "description": "Unacceptable MIME type"
+            },
+        }
+    })
+    def put(self):
+        if not request.accept_mimetypes.best_match(["application/json", "text/csv"]):
+            return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
+        # TODO catch error for bad filter
+        df = current_app.data.filter_dataframe(request.get_json()["filter"], include_uns=False)
+        if request.accept_mimetypes.best_match(['application/json']):
+            return make_response((jsonify(current_app.data.data_frame(df, axis=Axis.VAR))))
         # TODO support CSV
         else:
             return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
@@ -536,4 +632,5 @@ def get_api_resources():
     api.add_resource(DiffExpObsAPI, "/diffexp/obs")
     api.add_resource(AnnotationsVarAPI, "/annotations/var")
     api.add_resource(DataObsAPI, "/data/obs")
+    api.add_resource(DataVarAPI, "/data/var")
     return api

server/app/scanpy_engine/scanpy_engine.py

@@ -280,7 +280,7 @@ class ScanpyEngine(CXGDriver):
         return sorted(result, key=lambda gene: gene[0])
 
     # @cache.memoize()
-    def data_frame(self, df):
+    def data_frame(self, df, axis):
         """
         Retrieves data for each variable for observations in data frame
         :param df: from filter_cells, dataframe
@@ -289,9 +289,24 @@ class ScanpyEngine(CXGDriver):
             "obs": [cellid, var1 expression, var2 expression, ...],
         }
         """
-        var_index = df.var.index.tolist()
-        expression = DataFrame(df.X, index=df.obs.index)
-        return {
-            "var": var_index,
-            "obs": expression.reset_index().values.tolist()
-        }
+        var_idx = df.var.index.tolist()
+        obs_idx = df.obs.index.tolist()
+        values = df.X
+        df_shape = df.shape
+        if df_shape[0] == 1:
+            values = values[None, :]
+        elif df_shape[1] == 1:
+            values = values[:, None]
+        if axis == Axis.OBS:
+            expression = DataFrame(values, index=obs_idx)
+            result = {
+                "var": var_idx,
+                "obs": expression.reset_index().values.tolist()
+            }
+        else:
+            expression = DataFrame(values.T, index=var_idx)
+            result = {
+                "obs": obs_idx,
+                "var": expression.reset_index().values.tolist(),
+            }
+        return result

server/requirements.txt

@@ -7,5 +7,5 @@ Flask-RESTful==0.3.6
 flask-restful-swagger-2==0.35
 numpy==1.14.5
 pandas==0.23.1
-scanpy==1.0.4
+scanpy==1.3.1
 scipy==1.1.0

server/test/test_api.py

@@ -265,71 +265,81 @@ class EndPoints(unittest.TestCase):
         self.assertEqual(len(result_data["data"]), 2)
 
     def test_get_data(self):
-        endpoint = "data/obs"
-        query = "accept-type=application/json"
-        url = f"{URL_BASE}{endpoint}?{query}"
-        result = self.session.get(url)
-        self.assertEqual(result.status_code, 200)
-        result_data = result.json()
-        self.assertEqual(len(result_data["obs"]), 2638)
+        for axis in ["obs", "var"]:
+            endpoint = f"data/{axis}"
+            query = "accept-type=application/json"
+            url = f"{URL_BASE}{endpoint}?{query}"
+            result = self.session.get(url)
+            self.assertEqual(result.status_code, 200)
+            result_data = result.json()
+            self.assertEqual(len(result_data["obs"]), 2638)
 
     def test_data_mimetype_error(self):
-        endpoint = "data/obs"
-        query = "accept-type=xxx"
-        url = f"{URL_BASE}{endpoint}?{query}"
-        result = self.session.get(url)
-        self.assertEqual(result.status_code, 406)
-        # no accept type
-        url = f"{URL_BASE}{endpoint}"
-        result = self.session.get(url)
-        self.assertEqual(result.status_code, 406)
+        for axis in ["obs", "var"]:
+            endpoint = f"data/{axis}"
+            query = "accept-type=xxx"
+            url = f"{URL_BASE}{endpoint}?{query}"
+            result = self.session.get(url)
+            self.assertEqual(result.status_code, 406)
+            # no accept type
+            url = f"{URL_BASE}{endpoint}"
+            result = self.session.get(url)
+            self.assertEqual(result.status_code, 406)
 
     def test_data_filter(self):
-        endpoint = "data/obs"
-        query = "accept-type=application/json&obs:louvain=NK cells&obs:louvain=CD8 T cells&obs:n_counts=3000,*"
-        url = f"{URL_BASE}{endpoint}?{query}"
-        result = self.session.get(url)
-        self.assertEqual(result.status_code, 200)
-        result_data = result.json()
-        self.assertEqual(len(result_data["obs"]), 38)
+        for axis in ["obs", "var"]:
+            endpoint = f"data/{axis}"
+            query = "accept-type=application/json&obs:louvain=NK cells&obs:louvain=CD8 T cells&obs:n_counts=3000,*"
+            url = f"{URL_BASE}{endpoint}?{query}"
+            result = self.session.get(url)
+            self.assertEqual(result.status_code, 200)
+            result_data = result.json()
+            self.assertEqual(len(result_data["obs"]), 38)
 
     def test_data_put(self):
-        endpoint = "data/obs"
-        url = f"{URL_BASE}{endpoint}"
-        header = {"Accept": "application/json"}
-        obs_filter = {
-            "filter": {
-                "obs": {
-                    "annotation_value": [
-                        {"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
-                        {"name": "n_counts", "min": 3000},
-                    ],
-                    "index": [1, 99, [1000, 2000]]
+        for axis in ["obs", "var"]:
+            endpoint = f"data/{axis}"
+            url = f"{URL_BASE}{endpoint}"
+            header = {"Accept": "application/json"}
+            obs_filter = {
+                "filter": {
+                    "obs": {
+                        "annotation_value": [
+                            {"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
+                            {"name": "n_counts", "min": 3000},
+                        ],
+                        "index": [1, 99, [1000, 2000]]
+                    }
                 }
             }
-        }
-        result = self.session.put(url, headers=header, json=obs_filter)
-        self.assertEqual(result.status_code, 200)
-        result_data = result.json()
-        self.assertEqual(len(result_data["obs"]), 15)
+            result = self.session.put(url, headers=header, json=obs_filter)
+            self.assertEqual(result.status_code, 200)
+            result_data = result.json()
+            self.assertEqual(len(result_data["obs"]), 15)
 
     def test_data_put_single_var(self):
-        endpoint = "data/obs"
-        url = f"{URL_BASE}{endpoint}"
-        header = {"Accept": "application/json"}
-        var_filter = {
-            "filter": {
-                "var": {
-                    "annotation_value": [
-                        {"name": "name", "values": ["RER1"]},
-                    ]
+        for axis in ["obs", "var"]:
+            endpoint = f"data/{axis}"
+            url = f"{URL_BASE}{endpoint}"
+            header = {"Accept": "application/json"}
+            var_filter = {
+                "filter": {
+                    "var": {
+                        "annotation_value": [
+                            {"name": "name", "values": ["RER1"]},
+                        ]
+                    }
                 }
             }
-        }
-        result = self.session.put(url, headers=header, json=var_filter)
-        self.assertEqual(result.status_code, 200)
-        result_data = result.json()
-        self.assertEqual(len(result_data["obs"][0]), 2)
+            result = self.session.put(url, headers=header, json=var_filter)
+            self.assertEqual(result.status_code, 200)
+            result_data = result.json()
+            if axis == "obs":
+                self.assertEqual(len(result_data["obs"][0]), 2)
+                self.assertEqual(len(result_data["var"]), 1)
+            elif axis == "var":
+                self.assertEqual(len(result_data["obs"]), 2638)
+                self.assertEqual(len(result_data["var"][0]), 2639)
 
     def test_static(self):
         endpoint = "static"

server/test/test_scanpy_engine.py

@@ -206,9 +206,12 @@ class UtilTest(unittest.TestCase):
         self.assertEqual(len(result), 20)
 
     def test_data_frame(self):
-        data_frame = self.data.data_frame(self.data.data)
-        self.assertEqual(len(data_frame["var"]), 1838)
-        self.assertEqual(len(data_frame["obs"]), 2638)
+        data_frame_obs = self.data.data_frame(self.data.data, "obs")
+        self.assertEqual(len(data_frame_obs["var"]), 1838)
+        self.assertEqual(len(data_frame_obs["obs"]), 2638)
+        data_frame_var = self.data.data_frame(self.data.data, "var")
+        self.assertEqual(len(data_frame_var["var"]), 1838)
+        self.assertEqual(len(data_frame_var["obs"]), 2638)
 
     def test_filtered_data_frame(self):
         filter_ = {
@@ -221,9 +224,36 @@ class UtilTest(unittest.TestCase):
             }
         }
         data = self.data.filter_dataframe(filter_["filter"])
-        data_frame = self.data.data_frame(data)
-        self.assertEqual(len(data_frame["var"]), 1838)
-        self.assertEqual(len(data_frame["obs"]), 497)
+        data_frame_obs = self.data.data_frame(data, "obs")
+        self.assertEqual(len(data_frame_obs["var"]), 1838)
+        self.assertEqual(len(data_frame_obs["obs"]), 497)
+        self.assertEqual(type(data_frame_obs["obs"][0]), list)
+        self.assertEqual(type(data_frame_obs["var"][0]), int)
+        data_frame_var = self.data.data_frame(data, "var")
+        self.assertEqual(len(data_frame_var["var"]), 1838)
+        self.assertEqual(len(data_frame_var["obs"]), 497)
+        self.assertEqual(type(data_frame_var["var"][0]), list)
+        self.assertEqual(type(data_frame_var["obs"][0]), int)
+
+    def test_data_single_gene(self):
+        for axis in ["obs", "var"]:
+            filter_ = {
+                "filter": {
+                    "var": {
+                        "annotation_value": [
+                            {"name": "name", "values": ["RER1"]},
+                        ]
+                    }
+                }
+            }
+            data = self.data.filter_dataframe(filter_["filter"], include_uns=False)
+            data_frame_var = self.data.data_frame(data, axis)
+            if axis == "obs":
+                self.assertEqual(type(data_frame_var["var"][0]), int)
+                self.assertEqual(type(data_frame_var["obs"][0]), list)
+            elif axis == "var":
+                self.assertEqual(type(data_frame_var["obs"][0]), int)
+                self.assertEqual(type(data_frame_var["var"][0]), list)
 
     if __name__ == '__main__':
         unittest.main()