from http import HTTPStatus import pkg_resources from flask import ( Blueprint, current_app, jsonify, make_response, request ) from flask_restful_swagger_2 import Api, swagger, Resource from werkzeug.datastructures import ImmutableMultiDict from server.app.util.constants import Axis, DiffExpMode from server.app.util.filter import parse_filter, QueryStringError from server.app.util.models import FilterModel from server.app.util.utils import MimeTypeError, get_mime_type """ Sort order for routes 1. Initialize 2. Data & Metadata 3. Computation """ class SchemaAPI(Resource): @swagger.doc({ "summary": "get schema for dataframe and annotations", "tags": ["initialize"], "parameters": [], "responses": { "200": { "description": "schema", "examples": { "application/json": { "schema": { "dataframe": { "nObs": 383, "nVar": 19944, "type": "float32" }, "annotations": { "obs": [ {"name": "name", "type": "string"}, {"name": "tissue_type", "type": "string"}, {"name": "num_reads", "type": "int32"}, {"name": "sample_name", "type": "string"}, { "name": "clusters", "type": "categorical", "categories": [99, 1, "unknown cluster"] }, {"name": "QScore", "type": "float32"} ], "var": [ {"name": "name", "type": "string"}, {"name": "gene", "type": "string"} ] } } } } } } }) def get(self): return make_response(jsonify({"schema": current_app.data.schema}), HTTPStatus.OK) class ConfigAPI(Resource): @swagger.doc({ "summary": "Configuration information to assist in front-end adaptation" " to underlying engine, available functionality, interactive time limits, etc", "tags": ["initialize"], "parameters": [], "responses": { "200": { "description": "schema", "examples": { "application/json": { "config": { "features": [ {"method": "POST", "path": "/cluster/", "available": False}, { "method": "POST", "path": "/layout/obs", "available": True, "interactiveLimit": 10000 }, {"method": "POST", "path": "/layout/var", "available": False} ], "displayNames": { "engine": "ScanPy version 1.33", "dataset": "/home/joe/mouse/blorth.csv" }, } } } } } }) def get(self): config = { "config": { "features": [ {"method": "POST", "path": "/cluster/", **current_app.data.features["cluster"]}, {"method": "POST", "path": "/layout/obs", **current_app.data.features["layout"]["obs"]}, {"method": "POST", "path": "/layout/var", **current_app.data.features["layout"]["var"]}, {"method": "POST", "path": "/diffexp/", **current_app.data.features["diffexp"]}, ], "displayNames": { "engine": f"cellxgene Scanpy engine version {pkg_resources.get_distribution('cellxgene').version}", "dataset": current_app.config["DATASET_TITLE"] } } } return make_response(jsonify(config), HTTPStatus.OK) class AnnotationsObsAPI(Resource): @swagger.doc({ "summary": "Fetch annotations (metadata) for all observations.", "tags": ["annotations"], "parameters": [{ "in": "query", "name": "annotation-name", "type": "string", "description": "list of 1 or more annotation names" }], "responses": { "200": { "description": "annotations", "examples": { "application/json": { "names": [ "tissue_type", "sex", "num_reads", "clusters" ], "data": [ [0, "lung", "F", 39844, 99], [1, "heart", "M", 83, 1], [49, "spleen", None, 2, "unknown cluster"], ] } } }, "400": { "description": "one or more of the annotation-name identifiers were not associated with an " "annotation name" } } }) def get(self): fields = request.args.getlist("annotation-name", None) try: annotation_response = current_app.data.annotation(current_app.data.data, "obs", fields) except KeyError: return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST) return make_response(jsonify(annotation_response), HTTPStatus.OK) @swagger.doc({ "summary": "Fetch annotations (metadata) for filtered subset of observations.", "tags": ["annotations"], "parameters": [ { "in": "query", "name": "annotation-name", "type": "string", "description": "list of 1 or more annotation names" }, { "name": "filter", "description": "Complex Filter", "in": "body", "schema": FilterModel } ], "responses": { "200": { "description": "annotations", "examples": { "application/json": { "names": [ "tissue_type", "sex", "num_reads", "clusters" ], "data": [ [0, "lung", "F", 39844, 99], [1, "heart", "M", 83, 1], [49, "spleen", None, 2, "unknown cluster"], ] } } }, "400": { "description": "malformed filter or one or more of the annotation-name identifiers were" "not associated with an annotation name" } } }) def put(self): fields = request.args.getlist("annotation-name", None) try: df = current_app.data.filter_dataframe(request.get_json()["filter"], include_uns=False) except KeyError: return make_response("Malformed filter", HTTPStatus.BAD_REQUEST) try: annotation_response = current_app.data.annotation(df, "obs", fields) except KeyError: return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST) return make_response(jsonify(annotation_response), HTTPStatus.OK) class AnnotationsVarAPI(Resource): @swagger.doc({ "summary": "Fetch annotations (metadata) for all variables.", "tags": ["annotations"], "parameters": [{ "in": "query", "name": "annotation-name", "type": "string", "description": "list of 1 or more annotation names" }], "responses": { "200": { "description": "annotations", "examples": { "application/json": { "names": [ "name", "category" ], "data": [ [0, "ATAD3C", 1], [1, "RER1", None], [49, "S100B", 6] ] } } }, "400": { "description": "one or more of the annotation-name identifiers were not associated with an" " annotation name" } } }) def get(self): fields = request.args.getlist("annotation-name", None) try: annotation_response = current_app.data.annotation(current_app.data.data, "var", fields) except KeyError: return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST) return make_response(jsonify(annotation_response), HTTPStatus.OK) @swagger.doc({ "summary": "Fetch annotations (metadata) for filtered subset of variables.", "tags": ["annotations"], "parameters": [ { "in": "query", "name": "annotation-name", "type": "string", "description": "list of 1 or more annotation names" }, { "name": "filter", "description": "Complex Filter", "in": "body", "schema": FilterModel } ], "responses": { "200": { "description": "annotations", "examples": { "application/json": { "names": [ "name", "category" ], "data": [ [0, "ATAD3C", 1], [1, "RER1", None], [49, "S100B", 6] ] } } }, "400": { "description": "malformed filter or one or more of the annotation-name identifiers were" "not associated with an annotation name" } } }) def put(self): fields = request.args.getlist("annotation-name", None) try: df = current_app.data.filter_dataframe(request.get_json()["filter"], include_uns=False) except KeyError: return make_response("Malformed filter", HTTPStatus.BAD_REQUEST) try: annotation_response = current_app.data.annotation(df, "var", fields) except KeyError: return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST) return make_response(jsonify(annotation_response), HTTPStatus.OK) class DataObsAPI(Resource): @swagger.doc({ "summary": "Get data (expression values) from the dataframe.", "tags": ["data"], "parameters": [ { "in": "query", "name": "filter", "type": "string", "description": "axis:key:value" }, { "in": "query", "name": "accept-type", "type": "string", "description": "MIME type" }, ], "responses": { "200": { "description": "expression", "examples": { "application/json": { "var": [0, 20000], "obs": [ [1, 39483, 3902, 203, 0, 0, 28] ] } } }, "400": { "description": "Malformed filter" }, "406": { "description": "Unacceptable MIME type" }, } }) def get(self): accept_type = request.args.get("accept-type", None) # request.args is immutable args = dict(request.args) args.pop("accept-type", None) try: filter_ = parse_filter(ImmutableMultiDict(args), current_app.data.schema['annotations']) except QueryStringError as e: return make_response(e.message, HTTPStatus.BAD_REQUEST) try: df = current_app.data.filter_dataframe(filter_, include_uns=False) except KeyError: return make_response("malformed filter", HTTPStatus.BAD_REQUEST) # TODO support CSV try: # TODO store mime_type when more than one is supported get_mime_type(acceptable_types=["application/json"], query_param=accept_type, header=request.accept_mimetypes) except MimeTypeError as e: return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE) return make_response((jsonify(current_app.data.data_frame(df, axis=Axis.OBS))), HTTPStatus.OK) @swagger.doc({ "summary": "Get data (expression values) from the dataframe.", "tags": ["data"], "parameters": [ { 'name': 'filter', 'description': 'Complex Filter', 'in': 'body', 'schema': FilterModel } ], "responses": { "200": { "description": "expression", "examples": { "application/json": { "var": [0, 20000], "obs": [ [1, 39483, 3902, 203, 0, 0, 28] ] } } }, "400": { "description": "Malformed filter" }, "406": { "description": "Unacceptable MIME type" }, } }) def put(self): if not request.accept_mimetypes.best_match(["application/json", "text/csv"]): return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) try: df = current_app.data.filter_dataframe(request.get_json()["filter"], include_uns=False) except KeyError: return make_response("malformed filter", HTTPStatus.BAD_REQUEST) try: get_mime_type(acceptable_types=["application/json"], header=request.accept_mimetypes) except MimeTypeError as e: return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE) return make_response((jsonify(current_app.data.data_frame(df, axis=Axis.OBS))), HTTPStatus.OK) class DataVarAPI(Resource): @swagger.doc({ "summary": "Get data (expression values) from the dataframe.", "tags": ["data"], "parameters": [ { "in": "query", "name": "filter", "type": "string", "description": "axis:key:value" }, { "in": "query", "name": "accept-type", "type": "string", "description": "MIME type" }, ], "responses": { "200": { "description": "expression", "examples": { "application/json": { "obs": [0, 20000], "var": [ [1, 39483, 3902, 203, 0, 0, 28] ] } } }, "400": { "description": "Malformed filter" }, "406": { "description": "Unacceptable MIME type" }, } }) def get(self): accept_type = request.args.get("accept-type", None) # request.args is immutable args = dict(request.args) args.pop("accept-type", None) try: filter_ = parse_filter(ImmutableMultiDict(args), current_app.data.schema['annotations']) except QueryStringError as e: return make_response(e.message, HTTPStatus.BAD_REQUEST) try: df = current_app.data.filter_dataframe(filter_, include_uns=False) except KeyError: return make_response("malformed filter", HTTPStatus.BAD_REQUEST) try: get_mime_type(acceptable_types=["application/json"], query_param=accept_type, header=request.accept_mimetypes) except MimeTypeError as e: return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE) return make_response((jsonify(current_app.data.data_frame(df, axis=Axis.VAR))), HTTPStatus.OK) @swagger.doc({ "summary": "Get data (expression values) from the dataframe.", "tags": ["data"], "parameters": [ { 'name': 'filter', 'description': 'Complex Filter', 'in': 'body', 'schema': FilterModel } ], "responses": { "200": { "description": "expression", "examples": { "application/json": { "obs": [0, 20000], "var": [ [1, 39483, 3902, 203, 0, 0, 28] ] } } }, "400": { "description": "Malformed filter" }, "406": { "description": "Unacceptable MIME type" }, } }) def put(self): if not request.accept_mimetypes.best_match(["application/json", "text/csv"]): return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) try: df = current_app.data.filter_dataframe(request.get_json()["filter"], include_uns=False) except KeyError: return make_response("malformed filter", HTTPStatus.BAD_REQUEST) # TODO support CSV try: get_mime_type(acceptable_types=["application/json"], header=request.accept_mimetypes) except MimeTypeError as e: return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE) return make_response((jsonify(current_app.data.data_frame(df, axis=Axis.VAR))), HTTPStatus.OK) class DiffExpObsAPI(Resource): @swagger.doc({ "summary": "Generate differential expression (DE) statistics for two specified subsets of data, " "as indicated by the two provided observation complex filters", "tags": ["diffexp"], # TODO sort out params # "parameters": [ # # { # # "in": "body", # # "name": "mode", # # "type": "string", # # "required": True, # # "description": "topN or varFilter" # # }, # { # "in": "query", # "name": "count", # "type": "int32", # "description": "TopN mode: how many vars to return" # }, # { # "in": "body", # "name": "varFilter", # "schema": FilterModel, # "description": "varFilter: Complex filter, only var for which vars to return" # }, # { # "in": "body", # "name": "set1", # "schema": FilterModel, # "required": True, # "description": "Complex filter, only obs - observations in set1" # }, # { # "in": "body", # "name": "set2", # "schema": FilterModel, # "description": "Complex filter, only obs - observations in set2. If not included, inverse of set1." # }, # ], "responses": { "200": { "description": "Statistics are encoded as an array of arrays, with fields ordered as: " "varIndex, avgDiff, pVal, pValAdj, set1AvgExp, set2AvgExp", "examples": { "application/json": [ [328, -2.569489, 2.655706e-63, 3.642036e-57, 383.393, 583.9], [1250, -2.569489, 2.655706e-63, 3.642036e-57, 383.393, 583.9], ] } }, "400": { "description": "malformed filter" }, "403": { "description": "non-interactive request" }, "501": { "description": "diffexp is not implemented" } } }) def post(self): args = request.get_json() # confirm mode is present and legal try: mode = DiffExpMode(args["mode"]) except KeyError: return make_response("Error: mode is required", HTTPStatus.BAD_REQUEST) except ValueError: return make_response(f"Error: invalid mode option {args['mode']}", HTTPStatus.BAD_REQUEST) # Validate filters if mode == DiffExpMode.VAR_FILTER: if "varFilter" not in args: return make_response("varFilter is required when mode is set to varFilter ", HTTPStatus.BAD_REQUEST) if Axis.OBS in args["varFilter"]["filter"]: return make_response("Obs filter not allowed in varFilter", HTTPStatus.BAD_REQUEST) if "set1" not in args: return make_response("set1 is required.", HTTPStatus.BAD_REQUEST) if Axis.VAR in args["set1"]["filter"]: return make_response("Var filter not allowed for set1", HTTPStatus.BAD_REQUEST) # set2 if "set2" not in args: return make_response("Set2 as inverse of set1 is not implemented", HTTPStatus.NOT_IMPLEMENTED) if Axis.VAR in args["set2"]["filter"]: return make_response("Var filter not allowed for set2", HTTPStatus.BAD_REQUEST) set1_filter = args["set1"]["filter"] set2_filter = args.get("set2", {"filter": {}})["filter"] if "varFilter" in args: set1_filter[Axis.VAR] = args["varFilter"]["filter"][Axis.VAR] set2_filter[Axis.VAR] = args["varFilter"]["filter"][Axis.VAR] df1 = current_app.data.filter_dataframe(set1_filter, include_uns=False) # TODO inverse df2 = current_app.data.filter_dataframe(set2_filter, include_uns=False) # exceeds size limit if df1.shape[0] + df2.shape[0] > current_app.data.features["diffexp"]["interactiveLimit"]: return make_response("Non-interactive request", HTTPStatus.FORBIDDEN) # mode count = args.get("count", None) try: diffexp = current_app.data.diffexp(df1, df2, count) except ValueError as ve: return make_response(ve.message, HTTPStatus.BAD_REQUEST) return make_response(jsonify(diffexp), HTTPStatus.OK) class LayoutObsAPI(Resource): @swagger.doc({ "summary": "Get the default layout for all observations.", "tags": ["layout"], "parameters": [], "responses": { "200": { "description": "layout", "examples": { "application/json": { "layout": { "ndims": 2, "coordinates": [ [0, 0.284483, 0.983744], [1, 0.038844, 0.739444] ] } } } } } }) def get(self): return make_response((jsonify({"layout": current_app.data.layout(current_app.data.data)})), HTTPStatus.OK) @swagger.doc({ "summary": "Observation layout for filtered subset.", "tags": ["layout"], "parameters": [ { "name": "filter", "description": "Complex Filter", "in": "body", "schema": FilterModel } ], "responses": { "200": { "description": "layout", "examples": { "application/json": { "layout": { "ndims": 2, "coordinates": [ [0, 0.284483, 0.983744], [1, 0.038844, 0.739444] ] } } } }, "400": { "description": "Malformed filter" }, "403": { "description": "Non-interactive request" }, } }) def post(self): try: df = current_app.data.filter_dataframe(request.get_json()["filter"]) except KeyError: return make_response("Malformed filter", HTTPStatus.BAD_REQUEST) if len(df.obs.index) > current_app.data.features["layout"]["obs"]["interactiveLimit"]: return make_response("Non-interactive request", HTTPStatus.FORBIDDEN) return make_response((jsonify({"layout": current_app.data.layout(df)})), HTTPStatus.OK) def get_api_resources(): bp = Blueprint("api", __name__, url_prefix="/api/v0.2") api = Api(bp, add_api_spec_resource=False) # Initialization routes api.add_resource(SchemaAPI, "/schema") api.add_resource(ConfigAPI, "/config") # Data routes api.add_resource(AnnotationsObsAPI, "/annotations/obs") api.add_resource(AnnotationsVarAPI, "/annotations/var") api.add_resource(DataObsAPI, "/data/obs") api.add_resource(DataVarAPI, "/data/var") # Computation routes api.add_resource(DiffExpObsAPI, "/diffexp/obs") api.add_resource(LayoutObsAPI, "/layout/obs") return api