mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-09-21 15:38:12 +08:00
Bruce Martin
eeec842ad0
Move to new REST v0.2 communication between front and back-end. This is a first cut implementation which is functional, but will need follow-up enhancements for performance, error checking, etc. Protocol spec is in docs directory. * Add filtering via indexing * Using new filter specs Indexing working * Added filtering by annotation value * factor out common methods * Documentation * create enum for axis (obs/var) * Better description for filter's return * Add boolean to enumerated types * Augmented enum for scanpy axis * Create schema for annotations Based on datatype within scanpy/anndata + tests * remove obsolete schema parse script * Update rest api to remove old routes and add schema route * Separate development requirements * Warning for unsupported datatypes * include -r requirements.txt in dev * Merged downcast warnings * Fixed bug where names were NaNs Needed to include the index too when creating the series * Add config endpoint * Generate app features from CLI selections * Move features to driver * Add tests for schema * Clearer version wording * python3 version of super * version from engine to package level * move features to driver * Revise layout function to match the new spec * GET for layout/obs * PUT Layout (#211) * PUT Layout * Csweaver/annotations (#212) * Update scanpy engine to support the rest v0.2 annotation requests * GET endpoint for obs annotations + tests * Documentation * Test annotations in scanpy engine * Description for annotation-keys param * annotation->annotations * clarified return for annotations * Use URL query list for annotations fields * parse_filter parses v0.2 GET filters (#215) * parse_filter parses v0.2 GET filters * Don't allow index filters from query params * Better variable conversion * Parse filter improvements - uses default dict - renamed filter -> query_filter * Cleanup Tasks (#216) * Add test_api back into travis build * Do custom JSON encoding the correct way * Run cellxgene server in test setup * Cleanup new tests too * Option to bind to all interfaces (#225) app.run("0.0.0.0") instead of app.run("127.0.0.1") binds to all interfaces. Note: There are comments on the internet that says that the flask server is not up to the task of production serving. I don't think that such scalability concerns apply here, but I was able to get cellxgene working with twistd relatively easily, and we could switch to that if there are scalability concerns. Test plan: browsed to <ip>:5005/api/v0.2/config on a different host. * Add filtering via indexing * Using new filter specs Indexing working * Added filtering by annotation value * factor out common methods * Documentation * create enum for axis (obs/var) * Better description for filter's return * Add boolean to enumerated types * Augmented enum for scanpy axis * Create schema for annotations Based on datatype within scanpy/anndata + tests * remove obsolete schema parse script * Update rest api to remove old routes and add schema route * Separate development requirements * Warning for unsupported datatypes * include -r requirements.txt in dev * Merged downcast warnings * Fixed bug where names were NaNs Needed to include the index too when creating the series * Add config endpoint * Generate app features from CLI selections * Move features to driver * Add tests for schema * Clearer version wording * python3 version of super * version from engine to package level * move features to driver * Revise layout function to match the new spec * GET for layout/obs * PUT Layout (#211) * PUT Layout * Csweaver/annotations (#212) * Update scanpy engine to support the rest v0.2 annotation requests * GET endpoint for obs annotations + tests * Documentation * Test annotations in scanpy engine * Description for annotation-keys param * annotation->annotations * clarified return for annotations * Use URL query list for annotations fields * parse_filter parses v0.2 GET filters (#215) * parse_filter parses v0.2 GET filters * Don't allow index filters from query params * Better variable conversion * Parse filter improvements - uses default dict - renamed filter -> query_filter * Cleanup Tasks (#216) * Add test_api back into travis build * Do custom JSON encoding the correct way * Run cellxgene server in test setup * Cleanup new tests too * Option to bind to all interfaces (#225) app.run("0.0.0.0") instead of app.run("127.0.0.1") binds to all interfaces. Note: There are comments on the internet that says that the flask server is not up to the task of production serving. I don't think that such scalability concerns apply here, but I was able to get cellxgene working with twistd relatively easily, and we could switch to that if there are scalability concerns. Test plan: browsed to <ip>:5005/api/v0.2/config on a different host. * Fix merge errors - import warnings was improperly deleted - scanpy engine tests were totally wrong * Fix merge error with driver * PUT /annotations (#235) * Add query param for annotation name * fix descriptions, eliminate else clause * first cut at initial data load on rest 0.2 api * Annotation var (#248) * Fix bug strings are always objects in pandas * Add axis to annotation method * Add /annotation/var to REST api * Csweaver/expressiondata (#242) * Refactor expression method for REST v2 * Add message to QueryStringError * Fix range filters * Add GET route for /data * /data PUT route * rename expression to data_frame * clarification of error * Improve accept type handling * support all schema types for 0.2 REST API * remove REST 0.1 code; connect var annotations loading * config reducer; use config to set data set title; remove obsolete templating code for data set title * REST 0.2 expression conversion support * partial port of expression to REST 0.2 * diffexp (#273) * Add diffexp method to scanpy and test * Minor tweaks to diffexp Get a minimal working version to unblock FE development * Fixing things git deleted * cleanup print statements * Add index test * additional, partial REST 0.2 bring up of diffexp * Ignore unstructured annotations for data (#275) This is a temp hack, need to figure out how to include data.uns if there is only one gene * diffexp REST 0.2 port finish * ignore unstructured annotaitons on all routes except layout * correctly use varDataCache; maintain state during world rebuild * correct varDataCache use * temporarily disable all memoization * refinements to expression data caching * clear cell sets upon regraph/reset * update version of REST to 0.2 * Travis build fixes - comment out cache import - fix duplicate test name * Remove dependency from travis * clarify semantics of config variables * move generic action helpers into util
540 lines
20 KiB
Python
540 lines
20 KiB
Python
from http import HTTPStatus
|
|
import pkg_resources
|
|
|
|
from flask import (
|
|
Blueprint, current_app, jsonify, make_response, request
|
|
)
|
|
from flask_restful_swagger_2 import Api, swagger, Resource
|
|
from werkzeug.datastructures import ImmutableMultiDict
|
|
|
|
from server.app.util.constants import Axis, DiffExpMode
|
|
from server.app.util.filter import parse_filter, QueryStringError
|
|
from server.app.util.models import FilterModel
|
|
|
|
|
|
class SchemaAPI(Resource):
|
|
@swagger.doc({
|
|
"summary": "get schema for dataframe and annotations",
|
|
"tags": ["initialize"],
|
|
"parameters": [],
|
|
"responses": {
|
|
"200": {
|
|
"description": "schema",
|
|
"examples": {
|
|
"application/json": {
|
|
"schema": {
|
|
"dataframe": {
|
|
"nObs": 383,
|
|
"nVar": 19944,
|
|
"type": "float32"
|
|
},
|
|
"annotations": {
|
|
"obs": [
|
|
{"name": "name", "type": "string"},
|
|
{"name": "tissue_type", "type": "string"},
|
|
{"name": "num_reads", "type": "int32"},
|
|
{"name": "sample_name", "type": "string"},
|
|
{
|
|
"name": "clusters",
|
|
"type": "categorical",
|
|
"categories": [99, 1, "unknown cluster"]
|
|
},
|
|
{"name": "QScore", "type": "float32"}
|
|
],
|
|
"var": [
|
|
{"name": "name", "type": "string"},
|
|
{"name": "gene", "type": "string"}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
})
|
|
def get(self):
|
|
return make_response(jsonify({"schema": current_app.data.schema}), 200)
|
|
|
|
|
|
class ConfigAPI(Resource):
|
|
@swagger.doc({
|
|
"summary": "Configuration information to assist in front-end adaptation"
|
|
" to underlying engine, available functionality, interactive time limits, etc",
|
|
"tags": ["initialize"],
|
|
"parameters": [],
|
|
"responses": {
|
|
"200": {
|
|
"description": "schema",
|
|
"examples": {
|
|
"application/json": {
|
|
"config": {
|
|
"features": [
|
|
{"method": "POST", "path": "/cluster/", "available": False},
|
|
{
|
|
"method": "POST",
|
|
"path": "/layout/obs",
|
|
"available": True,
|
|
"interactiveLimit": 10000
|
|
},
|
|
{"method": "POST", "path": "/layout/var", "available": False}
|
|
|
|
],
|
|
"displayNames": {
|
|
"engine": "ScanPy version 1.33",
|
|
"dataset": "/home/joe/mouse/blorth.csv"
|
|
},
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
})
|
|
def get(self):
|
|
config = {
|
|
"config": {
|
|
"features": [
|
|
{"method": "POST", "path": "/cluster/", **current_app.data.features["cluster"]},
|
|
{"method": "POST", "path": "/layout/obs", **current_app.data.features["layout"]["obs"]},
|
|
{"method": "POST", "path": "/layout/var", **current_app.data.features["layout"]["var"]},
|
|
{"method": "POST", "path": "/diffexp/", **current_app.data.features["diffexp"]},
|
|
],
|
|
"displayNames": {
|
|
"engine": f"cellxgene Scanpy engine version {pkg_resources.get_distribution('cellxgene').version}",
|
|
"dataset": current_app.config["DATASET_TITLE"]
|
|
}
|
|
}
|
|
}
|
|
return make_response(jsonify(config), 200)
|
|
|
|
|
|
class LayoutObsAPI(Resource):
|
|
@swagger.doc({
|
|
"summary": "Get the default layout for all observations.",
|
|
"tags": ["layout"],
|
|
"parameters": [],
|
|
"responses": {
|
|
"200": {
|
|
"description": "layout",
|
|
"examples": {
|
|
"application/json": {
|
|
"layout": {
|
|
"ndims": 2,
|
|
"coordinates": [
|
|
[0, 0.284483, 0.983744],
|
|
[1, 0.038844, 0.739444]
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
})
|
|
def get(self):
|
|
return make_response((jsonify({"layout": current_app.data.layout(current_app.data.data)})))
|
|
|
|
@swagger.doc({
|
|
"summary": "Observation layout for filtered subset.",
|
|
"tags": ["layout"],
|
|
"parameters": [
|
|
{
|
|
"name": "filter",
|
|
"description": "Complex Filter",
|
|
"in": "body",
|
|
"schema": FilterModel
|
|
}
|
|
],
|
|
"responses": {
|
|
"200": {
|
|
"description": "layout",
|
|
"examples": {
|
|
"application/json": {
|
|
"layout": {
|
|
"ndims": 2,
|
|
"coordinates": [
|
|
[0, 0.284483, 0.983744],
|
|
[1, 0.038844, 0.739444]
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
})
|
|
def put(self):
|
|
df = current_app.data.filter_dataframe(request.get_json()["filter"])
|
|
return make_response((jsonify({"layout": current_app.data.layout(df)})))
|
|
|
|
|
|
class AnnotationsObsAPI(Resource):
|
|
@swagger.doc({
|
|
"summary": "Fetch annotations (metadata) for all observations.",
|
|
"tags": ["annotations"],
|
|
"parameters": [{
|
|
"in": "query",
|
|
"name": "annotation-name",
|
|
"type": "string",
|
|
"description": "list of 1 or more annotation names"
|
|
}],
|
|
"responses": {
|
|
"200": {
|
|
"description": "annotations",
|
|
"examples": {
|
|
"application/json": {
|
|
"names": [
|
|
"tissue_type", "sex", "num_reads", "clusters"
|
|
],
|
|
"data": [
|
|
[0, "lung", "F", 39844, 99],
|
|
[1, "heart", "M", 83, 1],
|
|
[49, "spleen", None, 2, "unknown cluster"],
|
|
|
|
]
|
|
}
|
|
|
|
}
|
|
}
|
|
}
|
|
})
|
|
def get(self):
|
|
fields = request.args.getlist("annotation-name", None)
|
|
try:
|
|
annotation_response = current_app.data.annotation(current_app.data.data, "obs", fields)
|
|
except KeyError:
|
|
return make_response(f"Error bad key in {fields}", 404)
|
|
return make_response(jsonify(annotation_response))
|
|
|
|
@swagger.doc({
|
|
"summary": "Fetch annotations (metadata) for filtered subset of observations.",
|
|
"tags": ["annotations"],
|
|
"parameters": [
|
|
{
|
|
"in": "query",
|
|
"name": "annotation-name",
|
|
"type": "string",
|
|
"description": "list of 1 or more annotation names"
|
|
},
|
|
{
|
|
"name": "filter",
|
|
"description": "Complex Filter",
|
|
"in": "body",
|
|
"schema": FilterModel
|
|
}
|
|
],
|
|
"responses": {
|
|
"200": {
|
|
"description": "annotations",
|
|
"examples": {
|
|
"application/json": {
|
|
"names": [
|
|
"tissue_type", "sex", "num_reads", "clusters"
|
|
],
|
|
"data": [
|
|
[0, "lung", "F", 39844, 99],
|
|
[1, "heart", "M", 83, 1],
|
|
[49, "spleen", None, 2, "unknown cluster"],
|
|
|
|
]
|
|
}
|
|
|
|
}
|
|
}
|
|
}
|
|
})
|
|
def put(self):
|
|
fields = request.args.getlist("annotation-name", None)
|
|
df = current_app.data.filter_dataframe(request.get_json()["filter"], include_uns=False)
|
|
try:
|
|
annotation_response = current_app.data.annotation(df, "obs", fields)
|
|
except KeyError:
|
|
return make_response(f"Error bad key in {fields}", 404)
|
|
return make_response(jsonify(annotation_response))
|
|
|
|
|
|
class AnnotationsVarAPI(Resource):
|
|
@swagger.doc({
|
|
"summary": "Fetch annotations (metadata) for all variables.",
|
|
"tags": ["annotations"],
|
|
"parameters": [{
|
|
"in": "query",
|
|
"name": "annotation-name",
|
|
"type": "string",
|
|
"description": "list of 1 or more annotation names"
|
|
}],
|
|
"responses": {
|
|
"200": {
|
|
"description": "annotations",
|
|
"examples": {
|
|
"application/json": {
|
|
"names": [
|
|
"name", "category"
|
|
],
|
|
"data": [
|
|
[0, "ATAD3C", 1],
|
|
[1, "RER1", None],
|
|
[49, "S100B", 6]
|
|
]
|
|
}
|
|
|
|
}
|
|
}
|
|
}
|
|
})
|
|
def get(self):
|
|
fields = request.args.getlist("annotation-name", None)
|
|
try:
|
|
annotation_response = current_app.data.annotation(current_app.data.data, "var", fields)
|
|
except KeyError:
|
|
return make_response(f"Error bad key in {fields}", 404)
|
|
return make_response(jsonify(annotation_response))
|
|
|
|
@swagger.doc({
|
|
"summary": "Fetch annotations (metadata) for filtered subset of variables.",
|
|
"tags": ["annotations"],
|
|
"parameters": [
|
|
{
|
|
"in": "query",
|
|
"name": "annotation-name",
|
|
"type": "string",
|
|
"description": "list of 1 or more annotation names"
|
|
},
|
|
{
|
|
"name": "filter",
|
|
"description": "Complex Filter",
|
|
"in": "body",
|
|
"schema": FilterModel
|
|
}
|
|
],
|
|
"responses": {
|
|
"200": {
|
|
"description": "annotations",
|
|
"examples": {
|
|
"application/json": {
|
|
"names": [
|
|
"name", "category"
|
|
],
|
|
"data": [
|
|
[0, "ATAD3C", 1],
|
|
[1, "RER1", None],
|
|
[49, "S100B", 6]
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
})
|
|
def put(self):
|
|
fields = request.args.getlist("annotation-name", None)
|
|
df = current_app.data.filter_dataframe(request.get_json()["filter"], include_uns=False)
|
|
try:
|
|
annotation_response = current_app.data.annotation(df, "var", fields)
|
|
except KeyError:
|
|
return make_response(f"Error bad key in {fields}", 404)
|
|
return make_response(jsonify(annotation_response))
|
|
|
|
|
|
class DiffExpObsAPI(Resource):
|
|
@swagger.doc({
|
|
"summary": "Generate differential expression (DE) statistics for two specified subsets of data, "
|
|
"as indicated by the two provided observation complex filters",
|
|
"tags": ["diffexp"],
|
|
# TODO sort out params
|
|
# "parameters": [
|
|
# # {
|
|
# # "in": "body",
|
|
# # "name": "mode",
|
|
# # "type": "string",
|
|
# # "required": True,
|
|
# # "description": "topN or varFilter"
|
|
# # },
|
|
# {
|
|
# "in": "query",
|
|
# "name": "count",
|
|
# "type": "int32",
|
|
# "description": "TopN mode: how many vars to return"
|
|
# },
|
|
# {
|
|
# "in": "body",
|
|
# "name": "varFilter",
|
|
# "schema": FilterModel,
|
|
# "description": "varFilter: Complex filter, only var for which vars to return"
|
|
# },
|
|
# {
|
|
# "in": "body",
|
|
# "name": "set1",
|
|
# "schema": FilterModel,
|
|
# "required": True,
|
|
# "description": "Complex filter, only obs - observations in set1"
|
|
# },
|
|
# {
|
|
# "in": "body",
|
|
# "name": "set2",
|
|
# "schema": FilterModel,
|
|
# "description": "Complex filter, only obs - observations in set2. If not included, inverse of set1."
|
|
# },
|
|
# ],
|
|
"responses": {
|
|
"200": {
|
|
"description": "Statistics are encoded as an array of arrays, with fields ordered as: "
|
|
"varIndex, avgDiff, pVal, pValAdj, set1AvgExp, set2AvgExp",
|
|
"examples": {
|
|
"application/json": [
|
|
[328, -2.569489, 2.655706e-63, 3.642036e-57, 383.393, 583.9],
|
|
[1250, -2.569489, 2.655706e-63, 3.642036e-57, 383.393, 583.9],
|
|
]
|
|
}
|
|
}
|
|
}
|
|
})
|
|
def post(self):
|
|
args = request.get_json()
|
|
# confirm mode is present and legal
|
|
try:
|
|
mode = DiffExpMode(args["mode"])
|
|
except KeyError:
|
|
return make_response("Error: mode is required", 400)
|
|
except ValueError:
|
|
return make_response(f"Error: invalid mode option {args['mode']}", 400)
|
|
# Validate filters
|
|
if mode == DiffExpMode.VAR_FILTER:
|
|
if "varFilter" not in args:
|
|
return make_response("varFilter is required when mode is set to varFilter ", 400)
|
|
if Axis.OBS in args["varFilter"]["filter"]:
|
|
return make_response("Obs filter not allowed in varFilter", 400)
|
|
if "set1" not in args:
|
|
return make_response("set1 is required.", 400)
|
|
if Axis.VAR in args["set1"]["filter"]:
|
|
return make_response("Var filter not allowed for set1", 400)
|
|
# set2
|
|
if "set2" not in args:
|
|
return make_response("Set2 as inverse of set1 is not implemented", 501)
|
|
if Axis.VAR in args["set2"]["filter"]:
|
|
return make_response("Var filter not allowed for set2", 400)
|
|
set1_filter = args["set1"]["filter"]
|
|
set2_filter = args.get("set2", {"filter": {}})["filter"]
|
|
if "varFilter" in args:
|
|
set1_filter[Axis.VAR] = args["varFilter"]["filter"][Axis.VAR]
|
|
set2_filter[Axis.VAR] = args["varFilter"]["filter"][Axis.VAR]
|
|
df1 = current_app.data.filter_dataframe(set1_filter, include_uns=False)
|
|
# TODO inverse
|
|
df2 = current_app.data.filter_dataframe(set2_filter, include_uns=False)
|
|
# exceeds size limit
|
|
if df1.shape[0] + df2.shape[0] > current_app.data.features["diffexp"]["interactiveLimit"]:
|
|
return make_response("Non-interactive request", 403)
|
|
# mode
|
|
count = args.get("count", None)
|
|
try:
|
|
diffexp = current_app.data.diffexp(df1, df2, count)
|
|
except ValueError as ve:
|
|
return make_response(ve.message, 400)
|
|
return make_response(jsonify(diffexp))
|
|
|
|
|
|
class DataObsAPI(Resource):
|
|
@swagger.doc({
|
|
"summary": "Get data (expression values) from the dataframe.",
|
|
"tags": ["data"],
|
|
"parameters": [
|
|
{
|
|
"in": "query",
|
|
"name": "filter",
|
|
"type": "string",
|
|
"description": "axis:key:value"
|
|
},
|
|
{
|
|
"in": "query",
|
|
"name": "accept-type",
|
|
"type": "string",
|
|
"description": "MIME type"
|
|
},
|
|
],
|
|
"responses": {
|
|
"200": {
|
|
"description": "expression",
|
|
"examples": {
|
|
"application/json": {
|
|
"var": [0, 20000],
|
|
"obs": [
|
|
[1, 39483, 3902, 203, 0, 0, 28]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"400": {
|
|
"description": "Malformed filter"
|
|
},
|
|
"406": {
|
|
"description": "Unacceptable MIME type"
|
|
},
|
|
}
|
|
})
|
|
def get(self):
|
|
# request.args is immutable
|
|
args = dict(request.args)
|
|
accept_type = args.pop("accept-type", None)
|
|
try:
|
|
filter_ = parse_filter(ImmutableMultiDict(args), current_app.data.schema['annotations'])
|
|
except QueryStringError as e:
|
|
return make_response(e.message, HTTPStatus.BAD_REQUEST)
|
|
df = current_app.data.filter_dataframe(filter_, include_uns=False)
|
|
if accept_type and accept_type[0] == "application/json":
|
|
return make_response((jsonify(current_app.data.data_frame(df))))
|
|
# TODO support CSV
|
|
else:
|
|
return make_response(f"Unsupported accept-type: {accept_type}", HTTPStatus.NOT_ACCEPTABLE)
|
|
|
|
@swagger.doc({
|
|
"summary": "Get data (expression values) from the dataframe.",
|
|
"tags": ["data"],
|
|
"parameters": [
|
|
{
|
|
'name': 'filter',
|
|
'description': 'Complex Filter',
|
|
'in': 'body',
|
|
'schema': FilterModel
|
|
}
|
|
],
|
|
"responses": {
|
|
"200": {
|
|
"description": "expression",
|
|
"examples": {
|
|
"application/json": {
|
|
"var": [0, 20000],
|
|
"obs": [
|
|
[1, 39483, 3902, 203, 0, 0, 28]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"400": {
|
|
"description": "Malformed filter"
|
|
},
|
|
"406": {
|
|
"description": "Unacceptable MIME type"
|
|
},
|
|
}
|
|
})
|
|
def put(self):
|
|
if not request.accept_mimetypes.best_match(["application/json", "text/csv"]):
|
|
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
|
|
# TODO catch error for bad filter
|
|
df = current_app.data.filter_dataframe(request.get_json()["filter"], include_uns=False)
|
|
if request.accept_mimetypes.best_match(['application/json']):
|
|
return make_response((jsonify(current_app.data.data_frame(df))))
|
|
# TODO support CSV
|
|
else:
|
|
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
|
|
|
|
|
|
def get_api_resources():
|
|
bp = Blueprint("api", __name__, url_prefix="/api/v0.2")
|
|
api = Api(bp, add_api_spec_resource=False)
|
|
api.add_resource(SchemaAPI, "/schema")
|
|
api.add_resource(ConfigAPI, "/config")
|
|
api.add_resource(LayoutObsAPI, "/layout/obs")
|
|
api.add_resource(AnnotationsObsAPI, "/annotations/obs")
|
|
api.add_resource(DiffExpObsAPI, "/diffexp/obs")
|
|
api.add_resource(AnnotationsVarAPI, "/annotations/var")
|
|
api.add_resource(DataObsAPI, "/data/obs")
|
|
return api
|