fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-10-02 04:38:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Move jsonification to engine level (#511)

Browse Source
This commit is contained in:
Charlotte Weaver
2018-12-18 21:14:24 -08:00
committed by GitHub
parent 6af7708d62
commit f0f7200f0b
10 changed files with 657 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/app/rest_api/rest.py
+274 -70
View File
@@ -6,11 +6,22 @@ from flask import Blueprint, current_app, jsonify, make_response, request
from flask_restful_swagger_2 import Api, swagger, Resource
from werkzeug.datastructures import ImmutableMultiDict
from server.app.util.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
from server.app.util.constants import (
Axis,
DiffExpMode,
JSON_MIMETYPE,
JSON_NaN_to_num_warning_msg,
)
from server.app.util.filter import parse_filter, QueryStringError
from server.app.util.models import FilterModel
from server.app.util.utils import get_mime_type
from server.app.util.errors import MimeTypeError, FilterError, InteractiveError, PrepareError
from server.app.util.errors import (
FilterError,
InteractiveError,
JSONEncodingValueError,
MimeTypeError,
PrepareError,
)
"""
Sort order for routes
@@ -32,7 +43,11 @@ class SchemaAPI(Resource):
"examples": {
"application/json": {
"schema": {
"dataframe": {"nObs": 383, "nVar": 19944, "type": "float32"},
"dataframe": {
"nObs": 383,
"nVar": 19944,
"type": "float32",
},
"annotations": {
"obs": [
{"name": "name", "type": "string"},
@@ -46,7 +61,10 @@ class SchemaAPI(Resource):
},
{"name": "QScore", "type": "float32"},
],
"var": [{"name": "name", "type": "string"}, {"name": "gene", "type": "string"}],
"var": [
{"name": "name", "type": "string"},
{"name": "gene", "type": "string"},
],
},
}
}
@@ -56,7 +74,9 @@ class SchemaAPI(Resource):
}
)
def get(self):
return make_response(jsonify({"schema": current_app.data.schema}), HTTPStatus.OK)
return make_response(
jsonify({"schema": current_app.data.schema}), HTTPStatus.OK
)
class ConfigAPI(Resource):
@@ -73,14 +93,22 @@ class ConfigAPI(Resource):
"application/json": {
"config": {
"features": [
{"method": "POST", "path": "/cluster/", "available": False},
{
"method": "POST",
"path": "/cluster/",
"available": False,
},
{
"method": "POST",
"path": "/layout/obs",
"available": True,
"interactiveLimit": 10000,
},
{"method": "POST", "path": "/layout/var", "available": False},
{
"method": "POST",
"path": "/layout/var",
"available": False,
},
],
"displayNames": {
"engine": "ScanPy version 1.33",
@@ -97,16 +125,34 @@ class ConfigAPI(Resource):
config = {
"config": {
"features": [
{"method": "POST", "path": "/cluster/", **current_app.data.features["cluster"]},
{"method": "POST", "path": "/layout/obs", **current_app.data.features["layout"]["obs"]},
{"method": "POST", "path": "/layout/var", **current_app.data.features["layout"]["var"]},
{"method": "POST", "path": "/diffexp/", **current_app.data.features["diffexp"]},
{
"method": "POST",
"path": "/cluster/",
**current_app.data.features["cluster"],
},
{
"method": "POST",
"path": "/layout/obs",
**current_app.data.features["layout"]["obs"],
},
{
"method": "POST",
"path": "/layout/var",
**current_app.data.features["layout"]["var"],
},
{
"method": "POST",
"path": "/diffexp/",
**current_app.data.features["diffexp"],
},
],
"displayNames": {
"engine": f"cellxgene Scanpy engine version {pkg_resources.get_distribution('cellxgene').version}",
"dataset": current_app.config["DATASET_TITLE"],
},
"parameters": {"max_category_items": current_app.data.max_category_items},
"parameters": {
"max_category_items": current_app.data.max_category_items
},
}
}
return make_response(jsonify(config), HTTPStatus.OK)
@@ -150,14 +196,17 @@ class AnnotationsObsAPI(Resource):
fields = request.args.getlist("annotation-name", None)
try:
annotation_response = current_app.data.annotation({}, "obs", fields)
return make_response(
annotation_response, HTTPStatus.OK, {"Content-Type": JSON_MIMETYPE}
)
except KeyError:
return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
try:
return make_response(jsonify(annotation_response), HTTPStatus.OK)
except ValueError as e:
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
@swagger.doc(
{
@@ -170,7 +219,12 @@ class AnnotationsObsAPI(Resource):
"type": "string",
"description": "list of 1 or more annotation names",
},
{"name": "filter", "description": "Complex Filter", "in": "body", "schema": FilterModel},
{
"name": "filter",
"description": "Complex Filter",
"in": "body",
"schema": FilterModel,
},
],
"responses": {
"200": {
@@ -196,17 +250,22 @@ class AnnotationsObsAPI(Resource):
def put(self):
fields = request.args.getlist("annotation-name", None)
try:
annotation_response = current_app.data.annotation(request.get_json()["filter"], "obs", fields)
annotation_response = current_app.data.annotation(
request.get_json()["filter"], "obs", fields
)
return make_response(
annotation_response, HTTPStatus.OK, {"Content-Type": JSON_MIMETYPE}
)
except KeyError:
return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
try:
return make_response(jsonify(annotation_response), HTTPStatus.OK)
except ValueError as e:
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
class AnnotationsVarAPI(Resource):
@@ -228,7 +287,11 @@ class AnnotationsVarAPI(Resource):
"examples": {
"application/json": {
"names": ["name", "category"],
"data": [[0, "ATAD3C", 1], [1, "RER1", None], [49, "S100B", 6]],
"data": [
[0, "ATAD3C", 1],
[1, "RER1", None],
[49, "S100B", 6],
],
}
},
},
@@ -243,14 +306,17 @@ class AnnotationsVarAPI(Resource):
fields = request.args.getlist("annotation-name", None)
try:
annotation_response = current_app.data.annotation({}, "var", fields)
return make_response(
annotation_response, HTTPStatus.OK, {"Content-Type": JSON_MIMETYPE}
)
except KeyError:
return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
try:
return make_response(jsonify(annotation_response), HTTPStatus.OK)
except ValueError as e:
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
@swagger.doc(
{
@@ -263,7 +329,12 @@ class AnnotationsVarAPI(Resource):
"type": "string",
"description": "list of 1 or more annotation names",
},
{"name": "filter", "description": "Complex Filter", "in": "body", "schema": FilterModel},
{
"name": "filter",
"description": "Complex Filter",
"in": "body",
"schema": FilterModel,
},
],
"responses": {
"200": {
@@ -271,7 +342,11 @@ class AnnotationsVarAPI(Resource):
"examples": {
"application/json": {
"names": ["name", "category"],
"data": [[0, "ATAD3C", 1], [1, "RER1", None], [49, "S100B", 6]],
"data": [
[0, "ATAD3C", 1],
[1, "RER1", None],
[49, "S100B", 6],
],
}
},
},
@@ -285,17 +360,22 @@ class AnnotationsVarAPI(Resource):
def put(self):
fields = request.args.getlist("annotation-name", None)
try:
annotation_response = current_app.data.annotation(request.get_json()["filter"], "var", fields)
annotation_response = current_app.data.annotation(
request.get_json()["filter"], "var", fields
)
return make_response(
annotation_response, HTTPStatus.OK, {"Content-Type": JSON_MIMETYPE}
)
except KeyError:
return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
except FilterError:
return make_response("Malformed filter", HTTPStatus.BAD_REQUEST)
try:
return make_response(jsonify(annotation_response), HTTPStatus.OK)
except ValueError as e:
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
class DataObsAPI(Resource):
@@ -304,13 +384,28 @@ class DataObsAPI(Resource):
"summary": "Get data (expression values) from the dataframe.",
"tags": ["data"],
"parameters": [
{"in": "query", "name": "filter", "type": "string", "description": "axis:key:value"},
{"in": "query", "name": "accept-type", "type": "string", "description": "MIME type"},
{
"in": "query",
"name": "filter",
"type": "string",
"description": "axis:key:value",
},
{
"in": "query",
"name": "accept-type",
"type": "string",
"description": "MIME type",
},
],
"responses": {
"200": {
"description": "expression",
"examples": {"application/json": {"var": [0, 20000], "obs": [[1, 39483, 3902, 203, 0, 0, 28]]}},
"examples": {
"application/json": {
"var": [0, 20000],
"obs": [[1, 39483, 3902, 203, 0, 0, 28]],
}
},
},
"400": {"description": "Malformed filter"},
"406": {"description": "Unacceptable MIME type"},
@@ -323,35 +418,57 @@ class DataObsAPI(Resource):
args = request.args.copy()
args.pop("accept-type", None)
try:
filter_ = parse_filter(ImmutableMultiDict(args), current_app.data.schema["annotations"])
filter_ = parse_filter(
ImmutableMultiDict(args), current_app.data.schema["annotations"]
)
except QueryStringError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
# TODO support CSV
try:
# TODO store mime_type when more than one is supported
get_mime_type(
acceptable_types=["application/json"], query_param=accept_type, header=request.accept_mimetypes
acceptable_types=["application/json"],
query_param=accept_type,
header=request.accept_mimetypes,
)
except MimeTypeError as e:
return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
try:
return make_response((jsonify(current_app.data.data_frame(filter_, axis=Axis.OBS))), HTTPStatus.OK)
return make_response(
current_app.data.data_frame(filter_, axis=Axis.OBS),
HTTPStatus.OK,
{"Content-Type": JSON_MIMETYPE},
)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except ValueError as e:
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
@swagger.doc(
{
"summary": "Get data (expression values) from the dataframe.",
"tags": ["data"],
"parameters": [{"name": "filter", "description": "Complex Filter", "in": "body", "schema": FilterModel}],
"parameters": [
{
"name": "filter",
"description": "Complex Filter",
"in": "body",
"schema": FilterModel,
}
],
"responses": {
"200": {
"description": "expression",
"examples": {"application/json": {"var": [0, 20000], "obs": [[1, 39483, 3902, 203, 0, 0, 28]]}},
"examples": {
"application/json": {
"var": [0, 20000],
"obs": [[1, 39483, 3902, 203, 0, 0, 28]],
}
},
},
"400": {"description": "Malformed filter"},
"406": {"description": "Unacceptable MIME type"},
@@ -360,21 +477,34 @@ class DataObsAPI(Resource):
)
def put(self):
if not request.accept_mimetypes.best_match(["application/json", "text/csv"]):
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
return make_response(
f"Unsupported MIME type '{request.accept_mimetypes}'",
HTTPStatus.NOT_ACCEPTABLE,
)
try:
get_mime_type(acceptable_types=["application/json"], header=request.accept_mimetypes)
get_mime_type(
acceptable_types=["application/json"], header=request.accept_mimetypes
)
except MimeTypeError as e:
return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
try:
return make_response(
(jsonify(current_app.data.data_frame(request.get_json()["filter"], axis=Axis.OBS))), HTTPStatus.OK
(
current_app.data.data_frame(
request.get_json()["filter"], axis=Axis.OBS
)
),
HTTPStatus.OK,
{"Content-Type": JSON_MIMETYPE},
)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except ValueError as e:
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
class DataVarAPI(Resource):
@@ -383,13 +513,28 @@ class DataVarAPI(Resource):
"summary": "Get data (expression values) from the dataframe.",
"tags": ["data"],
"parameters": [
{"in": "query", "name": "filter", "type": "string", "description": "axis:key:value"},
{"in": "query", "name": "accept-type", "type": "string", "description": "MIME type"},
{
"in": "query",
"name": "filter",
"type": "string",
"description": "axis:key:value",
},
{
"in": "query",
"name": "accept-type",
"type": "string",
"description": "MIME type",
},
],
"responses": {
"200": {
"description": "expression",
"examples": {"application/json": {"obs": [0, 20000], "var": [[1, 39483, 3902, 203, 0, 0, 28]]}},
"examples": {
"application/json": {
"obs": [0, 20000],
"var": [[1, 39483, 3902, 203, 0, 0, 28]],
}
},
},
"400": {"description": "Malformed filter"},
"406": {"description": "Unacceptable MIME type"},
@@ -402,33 +547,55 @@ class DataVarAPI(Resource):
args = request.args.copy()
args.pop("accept-type", None)
try:
filter_ = parse_filter(ImmutableMultiDict(args), current_app.data.schema["annotations"])
filter_ = parse_filter(
ImmutableMultiDict(args), current_app.data.schema["annotations"]
)
except QueryStringError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
try:
get_mime_type(
acceptable_types=["application/json"], query_param=accept_type, header=request.accept_mimetypes
acceptable_types=["application/json"],
query_param=accept_type,
header=request.accept_mimetypes,
)
except MimeTypeError as e:
return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
try:
return make_response((jsonify(current_app.data.data_frame(filter_, axis=Axis.VAR))), HTTPStatus.OK)
return make_response(
current_app.data.data_frame(filter_, axis=Axis.VAR),
HTTPStatus.OK,
{"Content-Type": JSON_MIMETYPE},
)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except ValueError as e:
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
@swagger.doc(
{
"summary": "Get data (expression values) from the dataframe.",
"tags": ["data"],
"parameters": [{"name": "filter", "description": "Complex Filter", "in": "body", "schema": FilterModel}],
"parameters": [
{
"name": "filter",
"description": "Complex Filter",
"in": "body",
"schema": FilterModel,
}
],
"responses": {
"200": {
"description": "expression",
"examples": {"application/json": {"obs": [0, 20000], "var": [[1, 39483, 3902, 203, 0, 0, 28]]}},
"examples": {
"application/json": {
"obs": [0, 20000],
"var": [[1, 39483, 3902, 203, 0, 0, 28]],
}
},
},
"400": {"description": "Malformed filter"},
"406": {"description": "Unacceptable MIME type"},
@@ -437,22 +604,35 @@ class DataVarAPI(Resource):
)
def put(self):
if not request.accept_mimetypes.best_match(["application/json", "text/csv"]):
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
return make_response(
f"Unsupported MIME type '{request.accept_mimetypes}'",
HTTPStatus.NOT_ACCEPTABLE,
)
# TODO support CSV
try:
get_mime_type(acceptable_types=["application/json"], header=request.accept_mimetypes)
get_mime_type(
acceptable_types=["application/json"], header=request.accept_mimetypes
)
except MimeTypeError as e:
return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
try:
return make_response(
(jsonify(current_app.data.data_frame(request.get_json()["filter"], axis=Axis.VAR))), HTTPStatus.OK
(
current_app.data.data_frame(
request.get_json()["filter"], axis=Axis.VAR
)
),
HTTPStatus.OK,
{"Content-Type": JSON_MIMETYPE},
)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except ValueError as e:
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
class DiffExpObsAPI(Resource):
@@ -522,23 +702,35 @@ class DiffExpObsAPI(Resource):
except KeyError:
return make_response("Error: mode is required", HTTPStatus.BAD_REQUEST)
except ValueError:
return make_response(f"Error: invalid mode option {args['mode']}", HTTPStatus.BAD_REQUEST)
return make_response(
f"Error: invalid mode option {args['mode']}", HTTPStatus.BAD_REQUEST
)
# Validate filters
if mode == DiffExpMode.VAR_FILTER or "varFilter" in args:
# not NOT_IMPLEMENTED
return make_response("mode=varfilter not implemented", HTTPStatus.NOT_IMPLEMENTED)
return make_response(
"mode=varfilter not implemented", HTTPStatus.NOT_IMPLEMENTED
)
if mode == DiffExpMode.TOP_N and "count" not in args:
return make_response("mode=topN requires a count parameter", HTTPStatus.BAD_REQUEST)
return make_response(
"mode=topN requires a count parameter", HTTPStatus.BAD_REQUEST
)
if "set1" not in args:
return make_response("set1 is required.", HTTPStatus.BAD_REQUEST)
if Axis.VAR in args["set1"]["filter"]:
return make_response("Var filter not allowed for set1", HTTPStatus.BAD_REQUEST)
return make_response(
"Var filter not allowed for set1", HTTPStatus.BAD_REQUEST
)
# set2
if "set2" not in args:
return make_response("Set2 as inverse of set1 is not implemented", HTTPStatus.NOT_IMPLEMENTED)
return make_response(
"Set2 as inverse of set1 is not implemented", HTTPStatus.NOT_IMPLEMENTED
)
if Axis.VAR in args["set2"]["filter"]:
return make_response("Var filter not allowed for set2", HTTPStatus.BAD_REQUEST)
return make_response(
"Var filter not allowed for set2", HTTPStatus.BAD_REQUEST
)
set1_filter = args["set1"]["filter"]
set2_filter = args.get("set2", {"filter": {}})["filter"]
@@ -549,18 +741,24 @@ class DiffExpObsAPI(Resource):
count = args.get("count", None)
try:
diffexp = current_app.data.diffexp_topN(
set1_filter, set2_filter, count, current_app.data.features["diffexp"]["interactiveLimit"]
set1_filter,
set2_filter,
count,
current_app.data.features["diffexp"]["interactiveLimit"],
)
return make_response(
diffexp, HTTPStatus.OK, {"Content-Type": JSON_MIMETYPE}
)
except (ValueError, FilterError) as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except InteractiveError:
return make_response("Non-interactive request", HTTPStatus.FORBIDDEN)
try:
return make_response(jsonify(diffexp), HTTPStatus.OK)
except ValueError as e:
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
class LayoutObsAPI(Resource):
@@ -576,7 +774,10 @@ class LayoutObsAPI(Resource):
"application/json": {
"layout": {
"ndims": 2,
"coordinates": [[0, 0.284_483, 0.983_744], [1, 0.038_844, 0.739_444]],
"coordinates": [
[0, 0.284_483, 0.983_744],
[1, 0.038_844, 0.739_444],
],
}
}
},
@@ -586,16 +787,19 @@ class LayoutObsAPI(Resource):
}
)
def get(self):
content_type = JSON_MIMETYPE
try:
layout = current_app.data.layout({})
except PrepareError as e:
return make_response(e.message, HTTPStatus.INTERNAL_SERVER_ERROR)
try:
return make_response((jsonify({"layout": layout})), HTTPStatus.OK)
except ValueError as e:
return make_response(layout, HTTPStatus.OK, {"Content-Type": content_type})
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
# @swagger.doc({
# "summary": "Observation layout for filtered subset.",
@@ -636,7 +840,7 @@ class LayoutObsAPI(Resource):
# filter = request.get_json()["filter"]
# interactive_limit = current_app.data.features["layout"]["obs"]["interactiveLimit"]
# layout = current_app.data.layout(filter, interactive_limit=interactive_limit)
# return make_response(jsonify({"layout": layout}), HTTPStatus.OK)
# return make_response(layout, HTTPStatus.OK, {"Content-Type": content_type})
# except FilterError as e:
# return make_response(e.message, HTTPStatus.BAD_REQUEST)
# except InteractiveError: