fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-21 22:08:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

add GET routes for expression data (#1387)

Browse Source 
* add GET routes for expression data

* fix comment typo
This commit is contained in:
Bruce Martin
2020-04-13 11:28:35 -07:00
committed by GitHub
parent 98c2a1359b
commit 136093d583
7 changed files with 323 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/src/actions/index.js
+83 -92
View File
@@ -1,4 +1,3 @@
import _ from "lodash";
import * as globals from "../globals";
import { Universe, MatrixFBS } from "../util/stateManager";
import * as Dataframe from "../util/dataframe";
@@ -6,7 +5,7 @@ import {
catchErrorsWrap,
doJsonRequest,
doBinaryRequest,
dispatchNetworkErrorMessageToUser
dispatchNetworkErrorMessageToUser,
} from "../util/actionHelpers";
import { PromiseLimit } from "../util/promiseLimit";
import { requestReembed, reembedResetWorldToUniverse } from "./reembed";
@@ -19,23 +18,23 @@ function obsAnnotationFetchAndLoad(dispatch, schema) {
const obsAnnotations = schema?.schema?.annotations?.obs ?? {};
const index = obsAnnotations.index ?? false;
const columns = (obsAnnotations.columns ?? []).filter(
col => col.name !== index
(col) => col.name !== index
);
const plimit = new PromiseLimit(4);
const plimit = new PromiseLimit(5);
return Promise.all(
columns.map(col =>
columns.map((col) =>
plimit.add(() => {
const path = `annotations/obs?annotation-name=${encodeURIComponent(
col.name
)}`;
const url = `${globals.API.prefix}${globals.API.version}${path}`;
return doBinaryRequest(url).then(buffer => {
return doBinaryRequest(url).then((buffer) => {
const df = Universe.matrixFBSToDataframe(buffer);
dispatch({
type: "universe: column load success",
dim: "obsAnnotations",
dataframe: df
dataframe: df,
});
});
})
@@ -52,20 +51,22 @@ function varAnnotationFetchAndLoad(dispatch, schema) {
const names = index ? [index] : [];
return Promise.all(
names
.map(name => {
.map((name) => {
const path = `annotations/var?annotation-name=${encodeURIComponent(
name
)}`;
const url = `${globals.API.prefix}${globals.API.version}${path}`;
return doBinaryRequest(url);
})
.map(rqst => rqst.then(buffer => Universe.matrixFBSToDataframe(buffer)))
.map(resp =>
resp.then(df =>
.map((rqst) =>
rqst.then((buffer) => Universe.matrixFBSToDataframe(buffer))
)
.map((resp) =>
resp.then((df) =>
dispatch({
type: "universe: column load success",
dim: "varAnnotations",
dataframe: df
dataframe: df,
})
)
)
@@ -77,25 +78,25 @@ return promise fetching layout we need
*/
function layoutFetchAndLoad(dispatch, schema) {
const embeddings = schema?.schema?.layout?.obs ?? [];
const embNames = embeddings.map(e => e.name);
const embNames = embeddings.map((e) => e.name);
const baseURL = `${globals.API.prefix}${globals.API.version}layout/obs`;
const plimit = new PromiseLimit(4);
const plimit = new PromiseLimit(5);
return Promise.all(
embNames.map(e =>
embNames.map((e) =>
plimit.add(() => {
const url = `${baseURL}?layout-name=${encodeURIComponent(e)}`;
return doBinaryRequest(url).then(buffer =>
return doBinaryRequest(url).then((buffer) =>
Universe.matrixFBSToDataframe(buffer)
);
})
)
).then(dfs => {
).then((dfs) => {
const df = Dataframe.Dataframe.empty().withColsFromAll(dfs);
dispatch({
type: "universe: column load success",
dim: "obsLayout",
dataframe: df
dataframe: df,
});
});
}
@@ -108,7 +109,7 @@ Bootstrap application with the initial data loading.
* /layout - all default layout
*/
const doInitialDataLoad = () =>
catchErrorsWrap(async dispatch => {
catchErrorsWrap(async (dispatch) => {
dispatch({ type: "initial data load start" });
try {
@@ -116,8 +117,8 @@ const doInitialDataLoad = () =>
Step 1 - config & schema, all JSON
*/
const requestJson = ["config", "schema"]
.map(r => `${globals.API.prefix}${globals.API.version}${r}`)
.map(url => doJsonRequest(url));
.map((r) => `${globals.API.prefix}${globals.API.version}${r}`)
.map((url) => doJsonRequest(url));
const stepOneResults = await Promise.all(requestJson);
/* set config defaults */
const config = { ...globals.configDefaults, ...stepOneResults[0].config };
@@ -125,11 +126,11 @@ const doInitialDataLoad = () =>
const universe = Universe.createUniverseFromResponse(config, schema);
dispatch({
type: "universe exists, but loading is still in progress",
universe
universe,
});
dispatch({
type: "configuration load complete",
config
config,
});
/*
@@ -137,7 +138,7 @@ const doInitialDataLoad = () =>
*/
await Promise.all([
layoutFetchAndLoad(dispatch, schema),
varAnnotationFetchAndLoad(dispatch, schema)
varAnnotationFetchAndLoad(dispatch, schema),
]);
/*
@@ -147,7 +148,7 @@ const doInitialDataLoad = () =>
dispatch({
type: "initial data load complete (universe exists)",
universe
universe,
});
} catch (error) {
dispatch({ type: "initial data load error", error });
@@ -164,7 +165,7 @@ const setWorldToSelection = () => (dispatch, getState) => {
type: "set World to current selection",
universe,
world,
crossfilter
crossfilter,
});
};
@@ -175,6 +176,11 @@ const dispatchExpressionErrors = (dispatch, res) => {
throw new Error(msg);
};
/* double URI encode - needed for query-param filters */
function dubEncURIComponent(s) {
return encodeURIComponent(encodeURIComponent(s));
}
/*
Fetch expression vectors for each gene in genes. This is NOT an action
function, but rather a helper to be called from an action helper that
@@ -188,51 +194,31 @@ async function _doRequestExpressionData(dispatch, getState, genes) {
const varIndexName = universe.schema.annotations.var.index;
/* helper for this function only */
const fetchData = async geneNames => {
const res = await fetch(
`${globals.API.prefix}${globals.API.version}data/var`,
{
method: "PUT",
body: JSON.stringify({
filter: {
var: {
annotation_value: [{ name: varIndexName, values: geneNames }]
}
}
}),
headers: new Headers({
accept: "application/octet-stream",
"Content-Type": "application/json"
}),
credentials: "include"
}
const fetchData = async (geneNames) => {
const query = geneNames
.map(
(g) =>
`var:${dubEncURIComponent(varIndexName)}=${dubEncURIComponent(g)}`
)
.join("&");
const url = `${globals.API.prefix}${globals.API.version}data/var?${query}`;
return doBinaryRequest(url).then((buffer) =>
// TODO: why convert to an Object and not a Dataframe?
Universe.convertDataFBStoObject(universe, buffer)
);
if (
!res.ok ||
res.headers.get("Content-Type") !== "application/octet-stream"
) {
// WILL throw
return dispatchExpressionErrors(dispatch, res);
}
const data = await res.arrayBuffer();
return Universe.convertDataFBStoObject(universe, data);
};
/* preload data already in cache */
let expressionData = _.transform(
genes,
(expData, g) => {
const data = universe.varData.col(g);
if (data) {
expData[g] = data.asArray();
}
},
{}
); // --> { gene: data }
let expressionData = genes.reduce((acc, g) => {
const data = universe.varData.col(g);
if (data) {
acc[g] = data.asArray();
}
return acc;
}, {}); // --> { gene: data }
/* make a list of genes for which we do not have data */
const genesToFetch = _.filter(genes, g => expressionData[g] === undefined);
const genesToFetch = genes.filter((g) => expressionData[g] === undefined);
dispatch({ type: "expression load start" });
@@ -242,7 +228,7 @@ async function _doRequestExpressionData(dispatch, getState, genes) {
const newExpressionData = await fetchData(genesToFetch);
expressionData = {
...expressionData,
...newExpressionData
...newExpressionData,
};
} catch (error) {
dispatch({ type: "expression load error", error });
@@ -264,19 +250,19 @@ function requestSingleGeneExpressionCountsForColoringPOST(gene) {
type: "color by expression",
gene,
data: {
[gene]: world.varData.col(gene).asArray()
}
[gene]: world.varData.col(gene).asArray(),
},
});
} catch (error) {
dispatch({
type: "get single gene expression for coloring error",
error
error,
});
}
};
}
const requestUserDefinedGene = gene => async (dispatch, getState) => {
const requestUserDefinedGene = (gene) => async (dispatch, getState) => {
dispatch({ type: "request user defined gene started" });
try {
await await _doRequestExpressionData(dispatch, getState, [gene]);
@@ -287,13 +273,13 @@ const requestUserDefinedGene = gene => async (dispatch, getState) => {
type: "request user defined gene success",
data: {
genes: [gene],
expression: world.varData.col(gene).asArray()
}
expression: world.varData.col(gene).asArray(),
},
});
} catch (error) {
return dispatch({
type: "request user defined gene error",
error
error,
});
}
};
@@ -315,7 +301,7 @@ const dispatchDiffExpErrors = (dispatch, response) => {
dispatchNetworkErrorMessageToUser(msg);
dispatch({
type: "request differential expression error",
error: new Error(msg)
error: new Error(msg),
});
}
}
@@ -353,15 +339,15 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
method: "POST",
headers: new Headers({
Accept: "application/json",
"Content-Type": "application/json"
"Content-Type": "application/json",
}),
body: JSON.stringify({
mode: "topN",
count: num_genes,
set1: { filter: { obs: { index: set1 } } },
set2: { filter: { obs: { index: set2 } } }
set2: { filter: { obs: { index: set2 } } },
}),
credentials: "include"
credentials: "include",
}
);
@@ -371,7 +357,7 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
const data = await res.json();
// result is [ [varIdx, ...], ... ]
const topNGenes = _.map(data, r =>
const topNGenes = data.map((r) =>
universe.varAnnotations.at(r[0], varIndexName)
);
@@ -379,17 +365,22 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
Kick off secondary action to fetch all of the expression data for the
topN expressed genes.
*/
await _doRequestExpressionData(dispatch, getState, topNGenes);
const plimit = new PromiseLimit(5);
await Promise.all(
topNGenes.map((gene) =>
plimit.add(() => _doRequestExpressionData(dispatch, getState, [gene]))
)
);
/* then send the success case action through */
return dispatch({
type: "request differential expression success",
data
data,
});
} catch (error) {
return dispatch({
type: "request differential expression error",
error
error,
});
}
};
@@ -399,7 +390,7 @@ const resetWorldToUniverse = () => (dispatch, getState) => {
reembedResetWorldToUniverse(dispatch, getState);
dispatch({
type: "reset World to eq Universe",
universe
universe,
});
};
@@ -409,12 +400,12 @@ const saveObsAnnotations = () => async (dispatch, getState) => {
const { dataCollectionNameIsReadOnly, dataCollectionName } = annotations;
dispatch({
type: "writable obs annotations - save started"
type: "writable obs annotations - save started",
});
const writableAnnotations = schema.annotations.obs.columns
.filter(s => s.writable)
.map(s => s.name);
.filter((s) => s.writable)
.map((s) => s.name);
const df = obsAnnotations.subset(null, writableAnnotations);
const matrix = MatrixFBS.encodeMatrixFBS(df);
try {
@@ -430,28 +421,28 @@ const saveObsAnnotations = () => async (dispatch, getState) => {
method: "PUT",
body: matrix,
headers: new Headers({
"Content-Type": "application/octet-stream"
"Content-Type": "application/octet-stream",
}),
credentials: "include"
credentials: "include",
}
);
if (res.ok) {
dispatch({
type: "writable obs annotations - save complete",
obsAnnotations
obsAnnotations,
});
} else {
dispatch({
type: "writable obs annotations - save error",
message: `HTTP error ${res.status} - ${res.statusText}`,
res
res,
});
}
} catch (error) {
dispatch({
type: "writable obs annotations - save error",
message: error.toString(),
error
error,
});
}
};
@@ -464,5 +455,5 @@ export default {
requestReembed,
resetWorldToUniverse,
saveObsAnnotations,
setWorldToSelection
setWorldToSelection,
};
client/src/reducers/controls.js
Vendored
+27 -26
View File
@@ -20,8 +20,6 @@ const Controls = (
scatterplotXXaccessor: null, // just easier to read
scatterplotYYaccessor: null,
graphRenderCounter: 0 /* integer as <Component key={graphRenderCounter} - a change in key forces a remount */,
__storedStateForCelllist1__: null /* will need procedural control of brush ie., brush.extent https://bl.ocks.org/micahstubbs/3cda05ca68cba260cb81 */,
__storedStateForCelllist2__: null
},
action,
nextSharedState,
@@ -62,7 +60,7 @@ const Controls = (
universeExists &&
embeddingsExist &&
varAnnotationsIndexExists
)
),
};
}
case "initial data load complete (universe exists)": {
@@ -71,38 +69,42 @@ const Controls = (
...state,
loading: false,
error: null,
resettingInterface: false
resettingInterface: false,
};
}
case "reset World to eq Universe": {
const [ newUserDefinedGenes, newDiffExpGenes ] = subsetAndResetGeneLists(state);
const [newUserDefinedGenes, newDiffExpGenes] = subsetAndResetGeneLists(
state
);
return {
...state,
resettingInterface: false,
userDefinedGenes: newUserDefinedGenes,
diffexpGenes: newDiffExpGenes
diffexpGenes: newDiffExpGenes,
};
}
case "set World to current selection": {
const [ newUserDefinedGenes, newDiffExpGenes ] = subsetAndResetGeneLists(state);
const [newUserDefinedGenes, newDiffExpGenes] = subsetAndResetGeneLists(
state
);
return {
...state,
loading: false,
error: null,
userDefinedGenes: newUserDefinedGenes,
diffexpGenes: newDiffExpGenes
diffexpGenes: newDiffExpGenes,
};
}
case "request user defined gene started": {
return {
...state,
userDefinedGenesLoading: true
userDefinedGenesLoading: true,
};
}
case "request user defined gene error": {
return {
...state,
userDefinedGenesLoading: false
userDefinedGenesLoading: false,
};
}
case "request user defined gene success": {
@@ -113,50 +115,49 @@ const Controls = (
return {
...state,
userDefinedGenes: _userDefinedGenes,
userDefinedGenesLoading: false
userDefinedGenesLoading: false,
};
}
case "request differential expression success": {
const { world } = prevSharedState;
const varIndexName = world.schema.annotations.var.index;
const _diffexpGenes = [];
action.data.forEach(d => {
action.data.forEach((d) => {
_diffexpGenes.push(world.varAnnotations.at(d[0], varIndexName));
});
return {
...state,
diffexpGenes: _diffexpGenes
diffexpGenes: _diffexpGenes,
};
}
case "clear differential expression": {
return {
...state,
diffexpGenes: []
diffexpGenes: [],
};
}
case "clear user defined gene": {
const { userDefinedGenes } = state;
const newUserDefinedGenes = _.filter(
userDefinedGenes,
d => d !== action.data
(d) => d !== action.data
);
return {
...state,
userDefinedGenes: newUserDefinedGenes
userDefinedGenes: newUserDefinedGenes,
};
}
case "clear all user defined genes": {
return {
...state,
userDefinedGenes: []
userDefinedGenes: [],
};
}
case "expression load error":
case "initial data load error": {
return {
...state,
loading: false,
error: action.error
error: action.error,
};
}
@@ -166,24 +167,24 @@ const Controls = (
case "change graph interaction mode":
return {
...state,
graphInteractionMode: action.data
graphInteractionMode: action.data,
};
case "change opacity deselected cells in 2d graph background":
return {
...state,
opacityForDeselectedCells: action.data
opacityForDeselectedCells: action.data,
};
case "increment graph render counter": {
const c = state.graphRenderCounter + 1;
return {
...state,
graphRenderCounter: c
graphRenderCounter: c,
};
}
case "interface reset started": {
return {
...state,
resettingInterface: true
resettingInterface: true,
};
}
@@ -193,18 +194,18 @@ const Controls = (
case "set scatterplot x":
return {
...state,
scatterplotXXaccessor: action.data
scatterplotXXaccessor: action.data,
};
case "set scatterplot y":
return {
...state,
scatterplotYYaccessor: action.data
scatterplotYYaccessor: action.data,
};
case "clear scatterplot":
return {
...state,
scatterplotXXaccessor: null,
scatterplotYYaccessor: null
scatterplotYYaccessor: null,
};
default:
client/src/util/actionHelpers.js
+24 -21
View File
@@ -6,7 +6,7 @@ import { postNetworkErrorToast } from "../components/framework/toasters";
dispatch an action error to the user. Currently we use
async toasts.
*/
export const dispatchNetworkErrorMessageToUser = message =>
export const dispatchNetworkErrorMessageToUser = (message) =>
postNetworkErrorToast(message);
/*
@@ -14,7 +14,7 @@ Catch unexpected errors and make sure we don't lose them!
*/
export function catchErrorsWrap(fn, dispatchToUser = false) {
return (dispatch, getState) => {
fn(dispatch, getState).catch(error => {
fn(dispatch, getState).catch((error) => {
console.error(error);
if (dispatchToUser) {
dispatchNetworkErrorMessageToUser(error.message);
@@ -29,30 +29,33 @@ Wrapper to perform async fetch with some modest error handling
and decoding.
*/
const doFetch = async (url, acceptType) => {
const res = await fetch(url, {
method: "get",
headers: new Headers({
Accept: acceptType
}),
credentials: "include"
});
if (res.ok && res.headers.get("Content-Type").includes(acceptType)) {
return res;
try {
const res = await fetch(url, {
method: "get",
headers: new Headers({
Accept: acceptType,
}),
credentials: "include",
});
if (res.ok && res.headers.get("Content-Type").includes(acceptType)) {
return res;
}
// else an error
const msg = `Unexpected HTTP response ${res.status}, ${res.statusText}`;
dispatchNetworkErrorMessageToUser(msg);
throw new Error(msg);
} catch (e) {
// network error
const msg = "Unexpected HTTP error";
dispatchNetworkErrorMessageToUser(msg);
throw e;
}
// else an error
let msg = `Unexpected HTTP response ${res.status}, ${res.statusText}`;
const body = await res.text();
if (body && body.length > 0) {
msg = `${msg} -- ${body}`;
}
dispatchNetworkErrorMessageToUser(msg);
throw new Error(msg);
};
/*
Wrapper to perform an async fetch and JSON decode response.
*/
export const doJsonRequest = async url => {
export const doJsonRequest = async (url) => {
const res = await doFetch(url, "application/json");
return res.json();
};
@@ -60,7 +63,7 @@ export const doJsonRequest = async url => {
/*
Wrapper to perform an async fetch for binary data.
*/
export const doBinaryRequest = async url => {
export const doBinaryRequest = async (url) => {
const res = await doFetch(url, "application/octet-stream");
return res.arrayBuffer();
};
server/app/app.py
+5
View File
@@ -198,6 +198,11 @@ class DataVarAPI(Resource):
def put(self, data_adaptor):
return common_rest.data_var_put(request, data_adaptor)
@cache_control(public=True, max_age=ONE_WEEK)
@rest_get_data_adaptor
def get(self, data_adaptor):
return common_rest.data_var_get(request, data_adaptor)
class DiffExpObsAPI(Resource):
@cache_control(no_store=True)
server/common/rest.py
+81
View File
@@ -3,6 +3,7 @@ from http import HTTPStatus
import copy
import logging
from flask import make_response, jsonify, current_app, abort
from werkzeug.urls import url_unquote
from server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
from server.common.errors import (
FilterError,
@@ -31,6 +32,70 @@ def abort_and_log(code, logmsg, loglevel=logging.DEBUG, include_exc_info=False):
return abort(code)
def _query_parameter_to_filter(args):
"""
Convert an annotation value filter, if present in the query args,
into the standard dict filter format used by internal code.
Query param filters look like: <axis>:name=value, where value
may be one of:
- a range, min,max, where either may be an open range by using an asterisc, eg, 10,*
- a value
Eg,
...?tissue=lung&obs:tissue=heart&obs:num_reads=1000,*
"""
filters = {
"obs": {},
"var": {},
}
# args has already been url-unquoted once. We assume double escaping
# on name and value.
try:
for key, value in args.items(multi=True):
axis, name = key.split(':')
if axis not in ("obs", "var"):
raise FilterError("unknown filter axis")
name = url_unquote(name)
current = filters[axis].setdefault(name, {"name": name})
val_split = value.split(',')
if len(val_split) == 1:
if 'min' in current or 'max' in current:
raise FilterError("do not mix range and value filters")
value = url_unquote(value)
values = current.setdefault("values", [])
values.append(value)
elif len(val_split) == 2:
if len(current) > 1:
raise FilterError("duplicate range specification")
min = url_unquote(val_split[0])
max = url_unquote(val_split[1])
if min != '*':
current["min"] = float(min)
if max != "*":
current["max"] = float(max)
if len(current) < 2:
raise FilterError("must specify at least min or max in range filter")
else:
raise FilterError("badly formated filter value")
except ValueError as e:
raise FilterError(str(e))
result = {}
for axis in ("obs", "var"):
axis_filter = filters[axis]
if len(axis_filter) > 0:
result[axis] = {
"annotation_value": [val for val in axis_filter.values()]
}
return result
def schema_get_helper(data_adaptor, annotations):
"""helper function to gather the schema from the data source and annotations"""
schema = data_adaptor.get_schema()
@@ -143,6 +208,22 @@ def data_var_put(request, data_adaptor):
return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
def data_var_get(request, data_adaptor):
preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
if preferred_mimetype != "application/octet-stream":
return abort(HTTPStatus.NOT_ACCEPTABLE)
try:
filter = _query_parameter_to_filter(request.args)
return make_response(
data_adaptor.data_frame_to_fbs_matrix(filter, axis=Axis.VAR),
HTTPStatus.OK,
{"Content-Type": "application/octet-stream"},
)
except (FilterError, ValueError, ExceedsLimitError) as e:
return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
def diffexp_obs_post(request, data_adaptor):
if not data_adaptor.config.diffexp__enable:
return abort(HTTPStatus.NOT_IMPLEMENTED)
server/test/test_api.py
+20
View File
@@ -206,6 +206,13 @@ class EndPoints(object):
result = self.session.put(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
def test_data_get_fbs(self):
endpoint = f"data/var"
url = f"{self.URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
def test_data_put_filter_fbs(self):
endpoint = f"data/var"
url = f"{self.URL_BASE}{endpoint}"
@@ -222,6 +229,19 @@ class EndPoints(object):
self.assertEqual(len(df["columns"]), df["n_cols"])
self.assertListEqual(df["col_idx"].tolist(), [0, 1, 4])
def test_data_get_filter_fbs(self):
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
query = f"var:{index_col_name}=SIK1"
endpoint = f"data/var"
url = f"{self.URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1)
def test_data_put_single_var(self):
endpoint = f"data/var"
url = f"{self.URL_BASE}{endpoint}"
server/test/test_filter.py
+83
View File
@@ -0,0 +1,83 @@
import unittest
from urllib.parse import parse_qs
from werkzeug.datastructures import MultiDict
from server.common.rest import _query_parameter_to_filter
from server.common.errors import FilterError
def _qsparse(qs):
""" emulate what Flask/Werkzeug do to our QS """
return MultiDict(parse_qs(qs))
class FilterParseTests(unittest.TestCase):
""" Test cases for various filter parsing """
def test_queryparam_to_filter_parse(self):
# categories
self.assertEqual(_query_parameter_to_filter(_qsparse("obs:foo=bar&var:baz=133&var:baz=A&obs:baz=foo")), {
"obs": {
"annotation_value": [
{"name": "foo", "values": ["bar"]},
{"name": "baz", "values": ["foo"]},
]
},
"var": {
"annotation_value": [
{"name": "baz", "values": ["133", "A"]}
]
}
})
# ranges
self.assertEqual(_query_parameter_to_filter(_qsparse("obs:A=1,99&obs:B=*,100&obs:C=0,*")), {
"obs": {
"annotation_value": [
{"name": "A", "min": 1, "max": 99.},
{"name": "B", "max": 100.},
{"name": "C", "min": 0.},
]
},
})
# combo
self.assertEqual(_query_parameter_to_filter(_qsparse("var:B=YES&var:A=1,99&var:B=NO")), {
"var": {
"annotation_value": [
{"name": "B", "values": ["YES", "NO"]},
{"name": "A", "min": 1., "max": 99.},
]
},
})
def test_queryparam_to_filter_escaping(self):
self.assertEqual(_query_parameter_to_filter(_qsparse("obs:var=%2521%252C%253AOK%253D&obs:A%2521=YO")), {
"obs": {
"annotation_value": [
{"name": "var", "values": ["!,:OK="]},
{"name": "A!", "values": ["YO"]},
],
},
})
def test_queryparam_to_filter_errors(self):
# should raise FilterError
filter_errors = [
"foo=bar", # no axis
"X=&Y=3", # no value
"X&Y=3", # no value
"moo:foo=bar", # bad axis
"obs:x=1,A", # non-numeric range
"var:X=1,2&var:X=3,4", # duplicate ranges
"var:Y=,",
"var:Y=2,",
"var:Y=,5",
"var:Y=*,",
"var:Y=,*",
"var:Y=*,*",
]
for qs in filter_errors:
with self.assertRaises(FilterError):
_query_parameter_to_filter(_qsparse(qs))