add GET routes for expression data (#1387)

* add GET routes for expression data

* fix comment typo

server/app/app.py

@@ -198,6 +198,11 @@ class DataVarAPI(Resource):
     def put(self, data_adaptor):
         return common_rest.data_var_put(request, data_adaptor)
 
+    @cache_control(public=True, max_age=ONE_WEEK)
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.data_var_get(request, data_adaptor)
+
 
 class DiffExpObsAPI(Resource):
     @cache_control(no_store=True)

server/common/rest.py

@@ -3,6 +3,7 @@ from http import HTTPStatus
 import copy
 import logging
 from flask import make_response, jsonify, current_app, abort
+from werkzeug.urls import url_unquote
 from server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
 from server.common.errors import (
     FilterError,
@@ -31,6 +32,70 @@ def abort_and_log(code, logmsg, loglevel=logging.DEBUG, include_exc_info=False):
     return abort(code)
 
 
+def _query_parameter_to_filter(args):
+    """
+    Convert an annotation value filter, if present in the query args,
+    into the standard dict filter format used by internal code.
+
+    Query param filters look like:  <axis>:name=value, where value
+    may be one of:
+        - a range, min,max, where either may be an open range by using an asterisc, eg, 10,*
+        - a value
+    Eg,
+        ...?tissue=lung&obs:tissue=heart&obs:num_reads=1000,*
+    """
+    filters = {
+        "obs": {},
+        "var": {},
+    }
+
+    # args has already been url-unquoted once.  We assume double escaping
+    # on name and value.
+    try:
+        for key, value in args.items(multi=True):
+            axis, name = key.split(':')
+            if axis not in ("obs", "var"):
+                raise FilterError("unknown filter axis")
+            name = url_unquote(name)
+            current = filters[axis].setdefault(name, {"name": name})
+
+            val_split = value.split(',')
+            if len(val_split) == 1:
+                if 'min' in current or 'max' in current:
+                    raise FilterError("do not mix range and value filters")
+                value = url_unquote(value)
+                values = current.setdefault("values", [])
+                values.append(value)
+
+            elif len(val_split) == 2:
+                if len(current) > 1:
+                    raise FilterError("duplicate range specification")
+                min = url_unquote(val_split[0])
+                max = url_unquote(val_split[1])
+                if min != '*':
+                    current["min"] = float(min)
+                if max != "*":
+                    current["max"] = float(max)
+                if len(current) < 2:
+                    raise FilterError("must specify at least min or max in range filter")
+
+            else:
+                raise FilterError("badly formated filter value")
+
+    except ValueError as e:
+        raise FilterError(str(e))
+
+    result = {}
+    for axis in ("obs", "var"):
+        axis_filter = filters[axis]
+        if len(axis_filter) > 0:
+            result[axis] = {
+                "annotation_value": [val for val in axis_filter.values()]
+            }
+
+    return result
+
+
 def schema_get_helper(data_adaptor, annotations):
     """helper function to gather the schema from the data source and annotations"""
     schema = data_adaptor.get_schema()
@@ -143,6 +208,22 @@ def data_var_put(request, data_adaptor):
         return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
 
 
+def data_var_get(request, data_adaptor):
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
+    if preferred_mimetype != "application/octet-stream":
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    try:
+        filter = _query_parameter_to_filter(request.args)
+        return make_response(
+            data_adaptor.data_frame_to_fbs_matrix(filter, axis=Axis.VAR),
+            HTTPStatus.OK,
+            {"Content-Type": "application/octet-stream"},
+        )
+    except (FilterError, ValueError, ExceedsLimitError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+
+
 def diffexp_obs_post(request, data_adaptor):
     if not data_adaptor.config.diffexp__enable:
         return abort(HTTPStatus.NOT_IMPLEMENTED)

server/test/test_api.py

@@ -206,6 +206,13 @@ class EndPoints(object):
         result = self.session.put(url, headers=header)
         self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
 
+    def test_data_get_fbs(self):
+        endpoint = f"data/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
+
     def test_data_put_filter_fbs(self):
         endpoint = f"data/var"
         url = f"{self.URL_BASE}{endpoint}"
@@ -222,6 +229,19 @@ class EndPoints(object):
         self.assertEqual(len(df["columns"]), df["n_cols"])
         self.assertListEqual(df["col_idx"].tolist(), [0, 1, 4])
 
+    def test_data_get_filter_fbs(self):
+        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        query = f"var:{index_col_name}=SIK1"
+        endpoint = f"data/var"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
+
     def test_data_put_single_var(self):
         endpoint = f"data/var"
         url = f"{self.URL_BASE}{endpoint}"

server/test/test_filter.py

@@ -0,0 +1,83 @@
+import unittest
+from urllib.parse import parse_qs
+from werkzeug.datastructures import MultiDict
+from server.common.rest import _query_parameter_to_filter
+from server.common.errors import FilterError
+
+
+def _qsparse(qs):
+    """ emulate what Flask/Werkzeug do to our QS """
+    return MultiDict(parse_qs(qs))
+
+
+class FilterParseTests(unittest.TestCase):
+    """ Test cases for various filter parsing """
+
+    def test_queryparam_to_filter_parse(self):
+        # categories
+        self.assertEqual(_query_parameter_to_filter(_qsparse("obs:foo=bar&var:baz=133&var:baz=A&obs:baz=foo")), {
+            "obs": {
+                "annotation_value": [
+                    {"name": "foo", "values": ["bar"]},
+                    {"name": "baz", "values": ["foo"]},
+                ]
+            },
+            "var": {
+                "annotation_value": [
+                    {"name": "baz", "values": ["133", "A"]}
+                ]
+            }
+        })
+
+        # ranges
+        self.assertEqual(_query_parameter_to_filter(_qsparse("obs:A=1,99&obs:B=*,100&obs:C=0,*")), {
+            "obs": {
+                "annotation_value": [
+                    {"name": "A", "min": 1, "max": 99.},
+                    {"name": "B", "max": 100.},
+                    {"name": "C", "min": 0.},
+                ]
+            },
+        })
+
+        # combo
+        self.assertEqual(_query_parameter_to_filter(_qsparse("var:B=YES&var:A=1,99&var:B=NO")), {
+            "var": {
+                "annotation_value": [
+                    {"name": "B", "values": ["YES", "NO"]},
+                    {"name": "A", "min": 1., "max": 99.},
+                ]
+            },
+        })
+
+    def test_queryparam_to_filter_escaping(self):
+        self.assertEqual(_query_parameter_to_filter(_qsparse("obs:var=%2521%252C%253AOK%253D&obs:A%2521=YO")), {
+            "obs": {
+                "annotation_value": [
+                    {"name": "var", "values": ["!,:OK="]},
+                    {"name": "A!", "values": ["YO"]},
+                ],
+            },
+        })
+
+    def test_queryparam_to_filter_errors(self):
+
+        # should raise FilterError
+        filter_errors = [
+            "foo=bar",  # no axis
+            "X=&Y=3",  # no value
+            "X&Y=3",  # no value
+            "moo:foo=bar",  # bad axis
+            "obs:x=1,A",  # non-numeric range
+            "var:X=1,2&var:X=3,4",  # duplicate ranges
+            "var:Y=,",
+            "var:Y=2,",
+            "var:Y=,5",
+            "var:Y=*,",
+            "var:Y=,*",
+            "var:Y=*,*",
+        ]
+
+        for qs in filter_errors:
+            with self.assertRaises(FilterError):
+                _query_parameter_to_filter(_qsparse(qs))