fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-10-01 08:38:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

Flatbuffer cleanup (#598)

Browse Source 
* dead code and route removal

* more dead code cleanup

* fix scanpy_engine tests

* lint

* add missing catch in filter parsing

* update scanpy NaN tests

* more fbs tests and dead test removal

* remove forced default for content type negotiation

* bit of cleanup

* more fbs test cleanup

* lint

* remove swagger

* swagger cleanup

* lint

* correctly handle lack of templates

* more dead code removal

* remove unused files

* fix dev build

* lint
This commit is contained in:
Bruce Martin
2019-02-19 08:50:29 -08:00
committed by GitHub
parent 4e67c645f8
commit 57c4e9ff33
16 changed files with 210 additions and 1663 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/app/app.py
-15
View File
@@ -4,7 +4,6 @@ from flask import Flask
from flask_caching import Cache
from flask_compress import Compress
from flask_cors import CORS
from flask_restful_swagger_2 import get_swagger_blueprint
from .rest_api.rest import get_api_resources
from .util.utils import Float32JSONEncoder
@@ -26,21 +25,7 @@ app.config.update(SECRET_KEY=SECRET_KEY)
# Application Data
data = None
# A list of swagger document objects
docs = []
resources = get_api_resources()
docs.append(resources.get_swagger_doc())
app.register_blueprint(webapp.bp)
app.register_blueprint(resources.blueprint)
app.register_blueprint(
get_swagger_blueprint(
docs,
"/api/swagger",
produces=["application/json"],
title="cellxgene rest api",
description="An API connecting ExpressionMatrix2 clustering algorithm to cellxgene",
)
)
app.add_url_rule("/", endpoint="index")
server/app/driver/driver.py
+2 -45
View File
@@ -42,45 +42,12 @@ class CXGDriver(metaclass=ABCMeta):
pass
@abstractmethod
def filter_dataframe(self, filter):
"""
Filter cells from data and return a subset of the data. They can operate on both obs and var dimension with
indexing and filtering by annotation value. Filters are combined with the and operator.
See REST specs for info on filter format:
https://github.com/chanzuckerberg/cellxgene/blob/master/docs/REST_API.md
:param filter: dictionary with filter params
:return: View into scanpy object with cells/genes filtered
"""
pass
@abstractmethod
def annotation(self, filter, axis, fields=None):
def annotation_to_fbs_matrix(self, axis, field=None):
"""
Gets annotation value for each observation
:param filter: filter: dictionary with filter params
:param axis: string obs or var
:param fields: list of keys for annotation to return, returns all annotation values if not set.
:return: dict: names - list of fields in order, data - list of lists or metadata
[observation ids, val1, val2...]
"""
pass
@abstractmethod
def annotation_to_fbs_matrix(self, axis, field=None):
""" Same as annotation(), except returns a flatbuffer, and does not support filtering. """
pass
@abstractmethod
def data_frame(self, filter, axis):
"""
Retrieves data for each variable for observations in data frame
:param filter: filter: dictionary with filter params
:param axis: string obs or var
:return: {
"var": list of variable ids,
"obs": [cellid, var1 expression, var2 expression, ...],
}
:return: flatbuffer: in fbs/matrix.fbs encoding
"""
pass
@@ -104,16 +71,6 @@ class CXGDriver(metaclass=ABCMeta):
"""
pass
@abstractmethod
def layout(self, filter, interactive_limit=None):
"""
Computes a n-d layout for cells through dimensionality reduction.
:param filter: filter: dictionary with filter params
:param interactive_limit: -- don't compute if total # genes in dataframes are larger than this
:return: [cellid, x, y, ...]
"""
pass
@abstractmethod
def layout_to_fbs_matrix(self, filter):
""" same as layout, except returns a flatbuffer """
server/app/rest_api/rest.py
+13 -674
View File
@@ -3,22 +3,17 @@ import pkg_resources
import warnings
from flask import Blueprint, current_app, jsonify, make_response, request
from flask_restful_swagger_2 import Api, swagger, Resource
from werkzeug.datastructures import ImmutableMultiDict
from flask_restful import Api, Resource
from server.app.util.constants import (
Axis,
DiffExpMode,
JSON_NaN_to_num_warning_msg,
)
from server.app.util.filter import parse_filter, QueryStringError
from server.app.util.models import FilterModel
from server.app.util.utils import get_mime_type
from server.app.util.errors import (
FilterError,
InteractiveError,
JSONEncodingValueError,
MimeTypeError,
PrepareError,
)
@@ -31,47 +26,6 @@ Sort order for routes
class SchemaAPI(Resource):
@swagger.doc(
{
"summary": "get schema for dataframe and annotations",
"tags": ["initialize"],
"parameters": [],
"responses": {
"200": {
"description": "schema",
"examples": {
"application/json": {
"schema": {
"dataframe": {
"nObs": 383,
"nVar": 19944,
"type": "float32",
},
"annotations": {
"obs": [
{"name": "name", "type": "string"},
{"name": "tissue_type", "type": "string"},
{"name": "num_reads", "type": "int32"},
{"name": "sample_name", "type": "string"},
{
"name": "clusters",
"type": "categorical",
"categories": [99, 1, "unknown cluster"],
},
{"name": "QScore", "type": "float32"},
],
"var": [
{"name": "name", "type": "string"},
{"name": "gene", "type": "string"},
],
},
}
}
},
}
},
}
)
def get(self):
return make_response(
jsonify({"schema": current_app.data.schema}), HTTPStatus.OK
@@ -79,47 +33,6 @@ class SchemaAPI(Resource):
class ConfigAPI(Resource):
@swagger.doc(
{
"summary": "Configuration information to assist in front-end adaptation"
" to underlying engine, available functionality, interactive time limits, etc",
"tags": ["initialize"],
"parameters": [],
"responses": {
"200": {
"description": "schema",
"examples": {
"application/json": {
"config": {
"features": [
{
"method": "POST",
"path": "/cluster/",
"available": False,
},
{
"method": "POST",
"path": "/layout/obs",
"available": True,
"interactiveLimit": 10000,
},
{
"method": "POST",
"path": "/layout/var",
"available": False,
},
],
"displayNames": {
"engine": "ScanPy version 1.33",
"dataset": "/home/joe/mouse/blorth.csv",
},
}
}
},
}
},
}
)
def get(self):
config = {
"config": {
@@ -158,51 +71,13 @@ class ConfigAPI(Resource):
class AnnotationsObsAPI(Resource):
@swagger.doc(
{
"summary": "Fetch annotations (metadata) for all observations.",
"tags": ["annotations"],
"parameters": [
{
"in": "query",
"name": "annotation-name",
"type": "string",
"description": "list of 1 or more annotation names",
}
],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": ["tissue_type", "sex", "num_reads", "clusters"],
"data": [
[0, "lung", "F", 39844, 99],
[1, "heart", "M", 83, 1],
[49, "spleen", None, 2, "unknown cluster"],
],
}
},
},
"400": {
"description": "one or more of the annotation-name identifiers were not associated with an "
"annotation name"
},
},
}
)
def get(self):
fields = request.args.getlist("annotation-name", None)
preferred_mimetype = request.accept_mimetypes.best_match(
["application/json", "application/octet-stream"],
"application/json"
["application/octet-stream"]
)
try:
if preferred_mimetype == "application/json":
return make_response(
current_app.data.annotation({}, "obs", fields), HTTPStatus.OK, {"Content-Type": "application/json"}
)
elif preferred_mimetype == "application/octet-stream":
if preferred_mimetype == "application/octet-stream":
return make_response(current_app.data.annotation_to_fbs_matrix("obs", fields),
HTTPStatus.OK,
{"Content-Type": "application/octet-stream"})
@@ -210,119 +85,18 @@ class AnnotationsObsAPI(Resource):
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
except KeyError:
return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
@swagger.doc(
{
"summary": "Fetch annotations (metadata) for filtered subset of observations.",
"tags": ["annotations"],
"parameters": [
{
"in": "query",
"name": "annotation-name",
"type": "string",
"description": "list of 1 or more annotation names",
},
{
"name": "filter",
"description": "Complex Filter",
"in": "body",
"schema": FilterModel,
},
],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": ["tissue_type", "sex", "num_reads", "clusters"],
"data": [
[0, "lung", "F", 39844, 99],
[1, "heart", "M", 83, 1],
[49, "spleen", None, 2, "unknown cluster"],
],
}
},
},
"400": {
"description": "malformed filter or one or more of the annotation-name identifiers were"
"not associated with an annotation name"
},
},
}
)
def put(self):
fields = request.args.getlist("annotation-name", None)
try:
annotation_response = current_app.data.annotation(
request.get_json()["filter"], "obs", fields
)
return make_response(
annotation_response, HTTPStatus.OK, {"Content-Type": "application/json"}
)
except KeyError:
return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
class AnnotationsVarAPI(Resource):
@swagger.doc(
{
"summary": "Fetch annotations (metadata) for all variables.",
"tags": ["annotations"],
"parameters": [
{
"in": "query",
"name": "annotation-name",
"type": "string",
"description": "list of 1 or more annotation names",
}
],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": ["name", "category"],
"data": [
[0, "ATAD3C", 1],
[1, "RER1", None],
[49, "S100B", 6],
],
}
},
},
"400": {
"description": "one or more of the annotation-name identifiers were not associated with an"
" annotation name"
},
},
}
)
def get(self):
fields = request.args.getlist("annotation-name", None)
preferred_mimetype = request.accept_mimetypes.best_match(
["application/json", "application/octet-stream"],
"application/json"
["application/octet-stream"]
)
try:
if preferred_mimetype == "application/json":
return make_response(current_app.data.annotation({}, "var", fields),
HTTPStatus.OK,
{"Content-Type": "application/json"})
elif preferred_mimetype == "application/octet-stream":
if preferred_mimetype == "application/octet-stream":
return make_response(current_app.data.annotation_to_fbs_matrix("var", fields),
HTTPStatus.OK,
{"Content-Type": "application/octet-stream"})
@@ -330,317 +104,22 @@ class AnnotationsVarAPI(Resource):
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
except KeyError:
return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
@swagger.doc(
{
"summary": "Fetch annotations (metadata) for filtered subset of variables.",
"tags": ["annotations"],
"parameters": [
{
"in": "query",
"name": "annotation-name",
"type": "string",
"description": "list of 1 or more annotation names",
},
{
"name": "filter",
"description": "Complex Filter",
"in": "body",
"schema": FilterModel,
},
],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": ["name", "category"],
"data": [
[0, "ATAD3C", 1],
[1, "RER1", None],
[49, "S100B", 6],
],
}
},
},
"400": {
"description": "malformed filter or one or more of the annotation-name identifiers were"
"not associated with an annotation name"
},
},
}
)
def put(self):
fields = request.args.getlist("annotation-name", None)
try:
annotation_response = current_app.data.annotation(
request.get_json()["filter"], "var", fields
)
return make_response(
annotation_response, HTTPStatus.OK, {"Content-Type": "application/json"}
)
except KeyError:
return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
except FilterError:
return make_response("Malformed filter", HTTPStatus.BAD_REQUEST)
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
class DataObsAPI(Resource):
@swagger.doc(
{
"summary": "Get data (expression values) from the dataframe.",
"tags": ["data"],
"parameters": [
{
"in": "query",
"name": "filter",
"type": "string",
"description": "axis:key:value",
},
{
"in": "query",
"name": "accept-type",
"type": "string",
"description": "MIME type",
},
],
"responses": {
"200": {
"description": "expression",
"examples": {
"application/json": {
"var": [0, 20000],
"obs": [[1, 39483, 3902, 203, 0, 0, 28]],
}
},
},
"400": {"description": "Malformed filter"},
"406": {"description": "Unacceptable MIME type"},
},
}
)
def get(self):
accept_type = request.args.get("accept-type", None)
# request.args is immutable
args = request.args.copy()
args.pop("accept-type", None)
try:
filter_ = parse_filter(
ImmutableMultiDict(args), current_app.data.schema["annotations"]
)
except QueryStringError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
# TODO support CSV
try:
# TODO store mime_type when more than one is supported
get_mime_type(
acceptable_types=["application/json"],
query_param=accept_type,
header=request.accept_mimetypes,
)
except MimeTypeError as e:
return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
try:
return make_response(
current_app.data.data_frame(filter_, axis=Axis.OBS),
HTTPStatus.OK,
{"Content-Type": "application/json"},
)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
@swagger.doc(
{
"summary": "Get data (expression values) from the dataframe.",
"tags": ["data"],
"parameters": [
{
"name": "filter",
"description": "Complex Filter",
"in": "body",
"schema": FilterModel,
}
],
"responses": {
"200": {
"description": "expression",
"examples": {
"application/json": {
"var": [0, 20000],
"obs": [[1, 39483, 3902, 203, 0, 0, 28]],
}
},
},
"400": {"description": "Malformed filter"},
"406": {"description": "Unacceptable MIME type"},
},
}
)
def put(self):
if not request.accept_mimetypes.best_match(["application/json", "text/csv"]):
return make_response(
f"Unsupported MIME type '{request.accept_mimetypes}'",
HTTPStatus.NOT_ACCEPTABLE,
)
try:
get_mime_type(
acceptable_types=["application/json"], header=request.accept_mimetypes
)
except MimeTypeError as e:
return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
try:
return make_response(
(
current_app.data.data_frame(
request.get_json()["filter"], axis=Axis.OBS
)
),
HTTPStatus.OK,
{"Content-Type": "application/json"},
)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
class DataVarAPI(Resource):
@swagger.doc(
{
"summary": "Get data (expression values) from the dataframe.",
"tags": ["data"],
"parameters": [
{
"in": "query",
"name": "filter",
"type": "string",
"description": "axis:key:value",
},
{
"in": "query",
"name": "accept-type",
"type": "string",
"description": "MIME type",
},
],
"responses": {
"200": {
"description": "expression",
"examples": {
"application/json": {
"obs": [0, 20000],
"var": [[1, 39483, 3902, 203, 0, 0, 28]],
}
},
},
"400": {"description": "Malformed filter"},
"406": {"description": "Unacceptable MIME type"},
},
}
)
def get(self):
accept_type = request.args.get("accept-type", None)
# request.args is immutable
args = request.args.copy()
args.pop("accept-type", None)
try:
filter_ = parse_filter(
ImmutableMultiDict(args), current_app.data.schema["annotations"]
)
except QueryStringError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
try:
get_mime_type(
acceptable_types=["application/json"],
query_param=accept_type,
header=request.accept_mimetypes,
)
except MimeTypeError as e:
return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
try:
return make_response(
current_app.data.data_frame(filter_, axis=Axis.VAR),
HTTPStatus.OK,
{"Content-Type": "application/json"},
)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
@swagger.doc(
{
"summary": "Get data (expression values) from the dataframe.",
"tags": ["data"],
"parameters": [
{
"name": "filter",
"description": "Complex Filter",
"in": "body",
"schema": FilterModel,
}
],
"responses": {
"200": {
"description": "expression",
"examples": {
"application/json": {
"obs": [0, 20000],
"var": [[1, 39483, 3902, 203, 0, 0, 28]],
}
},
},
"400": {"description": "Malformed filter"},
"406": {"description": "Unacceptable MIME type"},
},
}
)
def put(self):
preferred_mimetype = request.accept_mimetypes.best_match(
["application/json", "application/octet-stream"],
"application/json"
["application/octet-stream"]
)
try:
if preferred_mimetype == "application/json":
return make_response(
(
current_app.data.data_frame(
request.get_json()["filter"], axis=Axis.VAR
)
),
HTTPStatus.OK,
{"Content-Type": "application/json"},
)
elif preferred_mimetype == "application/octet-stream":
if preferred_mimetype == "application/octet-stream":
filter_json = request.get_json()
filter = filter_json["filter"] if filter_json else None
return make_response(
current_app.data.data_frame_to_fbs_matrix(
request.get_json()["filter"], axis=Axis.VAR
filter, axis=Axis.VAR
),
HTTPStatus.OK,
{"Content-Type": "application/octet-stream"})
@@ -648,73 +127,11 @@ class DataVarAPI(Resource):
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
class DiffExpObsAPI(Resource):
@swagger.doc(
{
"summary": "Generate differential expression (DE) statistics for two specified subsets of data, "
"as indicated by the two provided observation complex filters",
"tags": ["diffexp"],
# TODO sort out params
# "parameters": [
# # {
# # "in": "body",
# # "name": "mode",
# # "type": "string",
# # "required": True,
# # "description": "topN or varFilter"
# # },
# {
# "in": "query",
# "name": "count",
# "type": "int32",
# "description": "TopN mode: how many vars to return"
# },
# {
# "in": "body",
# "name": "varFilter",
# "schema": FilterModel,
# "description": "varFilter: Complex filter, only var for which vars to return"
# },
# {
# "in": "body",
# "name": "set1",
# "schema": FilterModel,
# "required": True,
# "description": "Complex filter, only obs - observations in set1"
# },
# {
# "in": "body",
# "name": "set2",
# "schema": FilterModel,
# "description": "Complex filter, only obs - observations in set2.
# If not included, inverse of set1."
# },
# ],
"responses": {
"200": {
"description": "Statistics are encoded as an array of arrays, with fields ordered as: "
"varIndex, logfoldchange, pVal, pValAdj",
"examples": {
"application/json": [
[328, -2.569_489, 2.655_706e-63, 3.642_036e-57],
[1250, -2.569_489, 2.655_706e-63, 3.642_036e-57],
]
},
},
"400": {"description": "malformed filter"},
"403": {"description": "non-interactive request"},
"501": {"description": "diffexp is not implemented"},
},
}
)
def post(self):
args = request.get_json()
# confirm mode is present and legal
@@ -783,40 +200,12 @@ class DiffExpObsAPI(Resource):
class LayoutObsAPI(Resource):
@swagger.doc(
{
"summary": "Get the default layout for all observations.",
"tags": ["layout"],
"parameters": [],
"responses": {
"200": {
"description": "layout",
"examples": {
"application/json": {
"layout": {
"ndims": 2,
"coordinates": [
[0, 0.284_483, 0.983_744],
[1, 0.038_844, 0.739_444],
],
}
}
},
},
"400": {"description": "Data preparation error"},
},
}
)
def get(self):
preferred_mimetype = request.accept_mimetypes.best_match(
["application/json", "application/octet-stream"],
"application/json"
["application/octet-stream"]
)
try:
if preferred_mimetype == "application/json":
return make_response(current_app.data.layout({}), HTTPStatus.OK, {"Content-Type": "application/json"})
elif preferred_mimetype == "application/octet-stream":
if preferred_mimetype == "application/octet-stream":
return make_response(current_app.data.layout_to_fbs_matrix(),
HTTPStatus.OK,
{"Content-Type": "application/octet-stream"})
@@ -824,69 +213,19 @@ class LayoutObsAPI(Resource):
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
except PrepareError as e:
return make_response(e.message, HTTPStatus.INTERNAL_SERVER_ERROR)
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
# @swagger.doc({
# "summary": "Observation layout for filtered subset.",
# "tags": ["layout"],
# "parameters": [
# {
# "name": "filter",
# "description": "Complex Filter",
# "in": "body",
# "schema": FilterModel
# }
# ],
# "responses": {
# "200": {
# "description": "layout",
# "examples": {
# "application/json": {
# "layout": {
# "ndims": 2,
# "coordinates": [
# [0, 0.284483, 0.983744],
# [1, 0.038844, 0.739444]
# ]
# }
# }
# }
# },
# "400": {
# "description": "Malformed filter"
# },
# "403": {
# "description": "Non-interactive request"
# },
# }
# })
# def put(self):
# try:
# filter = request.get_json()["filter"]
# interactive_limit = current_app.data.features["layout"]["obs"]["interactiveLimit"]
# layout = current_app.data.layout(filter, interactive_limit=interactive_limit)
# return make_response(layout, HTTPStatus.OK, {"Content-Type": content_type})
# except FilterError as e:
# return make_response(e.message, HTTPStatus.BAD_REQUEST)
# except InteractiveError:
# return make_response("Non-interactive request", HTTPStatus.FORBIDDEN)
def get_api_resources():
bp = Blueprint("api", __name__, url_prefix="/api/v0.2")
api = Api(bp, add_api_spec_resource=False)
api = Api(bp)
# Initialization routes
api.add_resource(SchemaAPI, "/schema")
api.add_resource(ConfigAPI, "/config")
# Data routes
api.add_resource(AnnotationsObsAPI, "/annotations/obs")
api.add_resource(AnnotationsVarAPI, "/annotations/var")
api.add_resource(DataObsAPI, "/data/obs")
api.add_resource(DataVarAPI, "/data/var")
# Computation routes
api.add_resource(DiffExpObsAPI, "/diffexp/obs")
server/app/scanpy_engine/scanpy_engine.py
+1 -169
View File
@@ -1,16 +1,13 @@
import warnings
import numpy as np
from pandas import DataFrame
from pandas.core.dtypes.dtypes import CategoricalDtype
import scanpy.api as sc
from scipy import sparse
from server.app.driver.driver import CXGDriver
from server.app.util.constants import Axis, DEFAULT_TOP_N
from server.app.util.errors import (
FilterError,
InteractiveError,
JSONEncodingValueError,
PrepareError,
ScanpyFileError,
@@ -197,23 +194,6 @@ class ScanpyEngine(CXGDriver):
f"to solve this problem. "
)
def filter_dataframe(self, filter):
"""
Filter cells from data and return a subset of the data. They can operate on both obs and var dimension with
indexing and filtering by annotation value. Filters are combined with the and operator.
See REST specs for info on filter format:
# TODO update this link to swagger when it's done
https://docs.google.com/document/d/1Fxjp1SKtCk7l8QP9-7KAjGXL0eldi_qEnNT0NmlGzXI/edit#heading=h.8qc9q57amldx
:param filter: dictionary with filter params
:return: View into scanpy object with cells/genes filtered
"""
if not filter:
return self.data
obs_selector, var_selector = self._filter_to_mask(filter, use_slices=False)
data = self._slice(self.data, obs_selector, var_selector)
return data
@staticmethod
def _annotation_filter_to_mask(filter, d_axis, count):
mask = np.ones((count,), dtype=bool)
@@ -277,72 +257,6 @@ class ScanpyEngine(CXGDriver):
)
return obs_selector, var_selector
@staticmethod
def _slice(data, obs_selector=None, vars_selector=None):
"""
Slice date using any selector that the AnnData object
supprots for slicing. If selector is None, will not slice
on that axis.
This method exists to optimize filtering/slicing sparse data that has
access patterns which impact slicing performance.
https://docs.scipy.org/doc/scipy/reference/sparse.html
"""
prefer_row_access = (
sparse.isspmatrix_csr(data._X)
or sparse.isspmatrix_lil(data._X)
or sparse.isspmatrix_bsr(data._X)
)
if prefer_row_access:
# Row-major slicing
if obs_selector is not None:
data = data[obs_selector, :]
if vars_selector is not None:
data = data[:, vars_selector]
else:
# Col-major slicing
if vars_selector is not None:
data = data[:, vars_selector]
if obs_selector is not None:
data = data[obs_selector, :]
return data
def annotation(self, filter, axis, fields=None):
"""
Gets annotation value for each observation
:param filter: filter: dictionary with filter params
:param axis: string obs or var
:param fields: list of keys for annotation to return, returns all annotation values if not set.
:return: dict: names - list of fields in order, data - list of lists or metadata
[observation ids, val1, val2...]
"""
try:
obs_selector, var_selector = self._filter_to_mask(filter)
except (KeyError, IndexError) as e:
raise FilterError(f"Error parsing filter: {e}") from e
if axis == Axis.OBS:
obs = self.data.obs[obs_selector]
if not fields:
fields = obs.columns.tolist()
result = {
"names": fields,
"data": DataFrame(obs[fields]).to_records(index=True).tolist(),
}
else:
var = self.data.var[var_selector]
if not fields:
fields = var.columns.tolist()
result = {
"names": fields,
"data": DataFrame(var[fields]).to_records(index=True).tolist(),
}
try:
return jsonify_scanpy(result)
except ValueError:
raise JSONEncodingValueError("Error encoding annotations to JSON")
def annotation_to_fbs_matrix(self, axis, fields=None):
if axis == Axis.OBS:
df = self.data.obs
@@ -352,44 +266,6 @@ class ScanpyEngine(CXGDriver):
df = df[fields]
return encode_matrix_fbs(df, col_idx=df.columns)
def data_frame(self, filter, axis):
"""
Retrieves data for each variable for observations in data frame
:param filter: filter: dictionary with filter params
:param axis: string obs or var
:return: {
"var": list of variable ids,
"obs": [cellid, var1 expression, var2 expression, ...],
}
"""
try:
obs_selector, var_selector = self._filter_to_mask(filter)
except (KeyError, IndexError) as e:
raise FilterError(f"Error parsing filter: {e}") from e
_X = self.data._X[obs_selector, var_selector]
if sparse.issparse(_X):
_X = _X.toarray()
var_index_sliced = self.data.var.index[var_selector]
obs_index_sliced = self.data.obs.index[obs_selector]
if axis == Axis.OBS:
result = {
"var": var_index_sliced.tolist(),
"obs": DataFrame(_X, index=obs_index_sliced)
.to_records(index=True)
.tolist(),
}
else:
result = {
"obs": obs_index_sliced.tolist(),
"var": DataFrame(_X.T, index=var_index_sliced)
.to_records(index=True)
.tolist(),
}
try:
return jsonify_scanpy(result)
except ValueError:
raise JSONEncodingValueError("Error encoding dataframe to JSON")
def data_frame_to_fbs_matrix(self, filter, axis):
"""
Retrieves data 'X' and returns in a flatbuffer Matrix.
@@ -405,7 +281,7 @@ class ScanpyEngine(CXGDriver):
raise ValueError("Only VAR dimension access is supported")
try:
obs_selector, var_selector = self._filter_to_mask(filter, use_slices=False)
except (KeyError, IndexError) as e:
except (KeyError, IndexError, TypeError) as e:
raise FilterError(f"Error parsing filter: {e}") from e
if obs_selector is not None:
raise FilterError("filtering on obs unsupported")
@@ -440,50 +316,6 @@ class ScanpyEngine(CXGDriver):
"Error encoding differential expression to JSON"
)
def layout(self, filter, interactive_limit=None):
"""
Computes a n-d layout for cells through dimensionality reduction.
:param filter: filter: dictionary with filter params
:param interactive_limit: -- don't compute if total # genes in dataframes are larger than this
:return: [cellid, x, y, ...]
"""
try:
df = self.filter_dataframe(filter)
except (KeyError, IndexError) as e:
raise FilterError(f"Error parsing filter: {e}") from e
if interactive_limit and len(df.obs.index) > interactive_limit:
raise InteractiveError("Size data is too large for interactive computation")
# TODO Filtering cells is fine, but filtering genes does nothing because the neighbors are
# calculated using the original vars (geneset) and this doesn’t get updated when you use less.
# Need to recalculate neighbors (long) if user requests new layout filtered by var
# TODO for MVP we are pushing computation of layout to preprocessing and not allowing re-layout
# this will probably change after user feedback
# getattr(sc.tl, self.layout_method)(df, random_state=123)
try:
df_layout = df.obsm[f"X_{self.layout_method}"]
except ValueError as e:
raise PrepareError(
f"Layout has not been calculated using {self.layout_method}, "
f"please prepare your datafile and relaunch cellxgene"
) from e
normalized_layout = DataFrame(
(df_layout - df_layout.min()) / (df_layout.max() - df_layout.min()),
index=df.obs.index,
)
try:
return jsonify_scanpy(
{
"layout": {
"ndims": normalized_layout.shape[1],
"coordinates": normalized_layout.to_records(
index=True
).tolist(),
}
}
)
except ValueError:
raise JSONEncodingValueError("Error encoding layout to JSON")
def layout_to_fbs_matrix(self):
"""
Return the default 2-D layout for cells as a FBS Matrix.
server/app/util/filter.py
-86
View File
@@ -1,86 +0,0 @@
import json
from collections import defaultdict
from numpy import float32, int32
from server.app.util.constants import Axis
class QueryStringError(Exception):
def __init__(self, key, message):
self.key = key
self.message = message
def _convert_variable(datatype, variable):
"""
Convert variable to number (float/int)
Used for dataset metadata and for query string
:param datatype: type to convert to
:param variable (string or None): value of variable
:return: converted variable
:raises: AssertionError
"""
assert datatype in ["boolean", "categorical", "float32", "int32", "string"]
if variable is None:
return variable
if datatype == "int32":
variable = int32(variable)
elif datatype == "float32":
variable = float32(variable)
elif datatype == "boolean":
variable = json.loads(variable)
assert isinstance(variable, bool)
return variable
def parse_filter(query_filter, schema):
"""
The filter comes in as arguments from a GET request
For categorical metadata keys filter based on axis:key=value
For continuous metadata keys filter by axis:key=min,max
Either value can be replaced by a * To have only a minimum
value axis:key=min,* To have only a maximum value axis:key=*,max
They combine via AND so a cell's metadata would have to match every filter
The results is a matrix with the cells the pass the filter and at this point all the genes
:param query_filter: flask's request.args
:param schema: dictionary schema
:raises QueryStringError
:return:
"""
query = defaultdict(lambda: defaultdict(list))
for key in query_filter:
axis, annotation = key.split(":", 1)
try:
Axis(axis)
except ValueError:
raise QueryStringError(key, f"Error: key {key} not in metadata schema")
ann_filter = {"name": annotation}
for ann in schema[axis]:
if ann["name"] == annotation:
dtype = ann["type"]
break
else:
raise QueryStringError(key, f"Error: {annotation} not a valid annotation name")
if dtype in ["string", "categorical", "boolean"]:
ann_filter["values"] = [_convert_variable(dtype, i) for i in query_filter.getlist(key)]
else:
value = query_filter.get(key)
try:
min_, max_ = value.split(",")
except ValueError:
raise QueryStringError(key, f"Error: min,max format required for range for {annotation}, got {value}")
if min_ == "*":
min_ = None
if max_ == "*":
max_ = None
try:
ann_filter["min"] = _convert_variable(dtype, min_)
ann_filter["max"] = _convert_variable(dtype, max_)
except ValueError:
raise QueryStringError(key, f"Error: expected type {query[key]['type']} for key {key}, got {value}")
query[axis]["annotation_value"].append(ann_filter)
return query
server/app/util/models.py
-34
View File
@@ -1,34 +0,0 @@
from flask_restful_swagger_2 import Schema
class AnnotationModel(Schema):
type = "object"
description = "Filter by annotation key: value"
properties = {
"name": {"type": "string"},
# TODO update to OpenAPI v3.0 when a library is available that supports it
# Unfortunately 2.0 doesn't have a way to have a schema that accepts multiple types
# Overloading the type key with a list seems to work ok and makes it to the page
"values": {"type": "array", "items": {"type": ["float32", "string", "int32", "bool"]}},
"min": {"type": ["int32", "float32"]},
"max": {"type": ["int32", "float32"]},
}
required = ["name"]
class IndexModel(Schema):
type = "object"
description = "Filter by index of observation/variable ex. [0, 5, 15]"
properties = {"index": {"type": "array", "items": {"format": "int32", "type": "integer"}}}
class AxisModel(Schema):
type = "object"
description = "Axis of data -- obs or var"
properties = {"index": IndexModel, "annotation_value": AnnotationModel.array()}
class FilterModel(Schema):
type = "object"
description = "Complex filter"
properties = {"filter": {"type": "object", "properties": {"obs": AxisModel, "var": AxisModel}}}
server/app/util/utils.py
-30
View File
@@ -1,10 +1,6 @@
import json
from argparse import ArgumentTypeError
from numpy import float32, integer
from server.app.util.errors import MimeTypeError
class Float32JSONEncoder(json.JSONEncoder):
def __init__(self, *args, **kwargs):
@@ -30,31 +26,5 @@ def custom_format_warning(msg, *args, **kwargs):
return f"[cellxgene] Warning: {msg} \n"
def get_mime_type(
default="application/json", acceptable_types=["application/json", "text/csv"], query_param=None, header=None
):
mime_type = default
if query_param:
if query_param in acceptable_types:
mime_type = query_param
else:
raise MimeTypeError(f"Unsupported mime type {query_param} specified in query parameter 'accept-type'")
elif len(header):
mime_type = header.best_match(acceptable_types)
if not mime_type:
raise MimeTypeError(f"Unsupported mime type(s) {header} in HTTP Accept header")
return mime_type
def whole_number(value):
try:
value = int(value)
except ValueError as e:
raise ArgumentTypeError(f"{value} is not type int") from e
if value < 0:
raise ArgumentTypeError(f"{value} is not >= 0")
return value
def jsonify_scanpy(data):
return json.dumps(data, cls=Float32JSONEncoder, allow_nan=False)
server/app/web/templates/swagger.html
-97
View File
@@ -1,97 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>cellxgene REST API - Swagger definition</title>
<link href="https://fonts.googleapis.com/css?family=Open+Sans:400,700|Source+Code+Pro:300,600|Titillium+Web:400,600,700"
rel="stylesheet">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.2.1/swagger-ui.css"
crossorigin="anonymous"/>
<style>
html {
box-sizing: border-box;
overflow: -moz-scrollbars-vertical;
overflow-y: scroll;
}
*,
*:before,
*:after {
box-sizing: inherit;
}
body {
margin: 0;
background: #fafafa;
}
</style>
</head>
<body>
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"
style="position:absolute;width:0;height:0">
<defs>
<symbol viewBox="0 0 20 20" id="unlocked">
<path d="M15.8 8H14V5.6C14 2.703 12.665 1 10 1 7.334 1 6 2.703 6 5.6V6h2v-.801C8 3.754 8.797 3 10 3c1.203 0 2 .754 2 2.199V8H4c-.553 0-1 .646-1 1.199V17c0 .549.428 1.139.951 1.307l1.197.387C5.672 18.861 6.55 19 7.1 19h5.8c.549 0 1.428-.139 1.951-.307l1.196-.387c.524-.167.953-.757.953-1.306V9.199C17 8.646 16.352 8 15.8 8z"></path>
</symbol>
<symbol viewBox="0 0 20 20" id="locked">
<path d="M15.8 8H14V5.6C14 2.703 12.665 1 10 1 7.334 1 6 2.703 6 5.6V8H4c-.553 0-1 .646-1 1.199V17c0 .549.428 1.139.951 1.307l1.197.387C5.672 18.861 6.55 19 7.1 19h5.8c.549 0 1.428-.139 1.951-.307l1.196-.387c.524-.167.953-.757.953-1.306V9.199C17 8.646 16.352 8 15.8 8zM12 8H8V5.199C8 3.754 8.797 3 10 3c1.203 0 2 .754 2 2.199V8z"/>
</symbol>
<symbol viewBox="0 0 20 20" id="close">
<path d="M14.348 14.849c-.469.469-1.229.469-1.697 0L10 11.819l-2.651 3.029c-.469.469-1.229.469-1.697 0-.469-.469-.469-1.229 0-1.697l2.758-3.15-2.759-3.152c-.469-.469-.469-1.228 0-1.697.469-.469 1.228-.469 1.697 0L10 8.183l2.651-3.031c.469-.469 1.228-.469 1.697 0 .469.469.469 1.229 0 1.697l-2.758 3.152 2.758 3.15c.469.469.469 1.229 0 1.698z"/>
</symbol>
<symbol viewBox="0 0 20 20" id="large-arrow">
<path d="M13.25 10L6.109 2.58c-.268-.27-.268-.707 0-.979.268-.27.701-.27.969 0l7.83 7.908c.268.271.268.709 0 .979l-7.83 7.908c-.268.271-.701.27-.969 0-.268-.269-.268-.707 0-.979L13.25 10z"/>
</symbol>
<symbol viewBox="0 0 20 20" id="large-arrow-down">
<path d="M17.418 6.109c.272-.268.709-.268.979 0s.271.701 0 .969l-7.908 7.83c-.27.268-.707.268-.979 0l-7.908-7.83c-.27-.268-.27-.701 0-.969.271-.268.709-.268.979 0L10 13.25l7.418-7.141z"/>
</symbol>
<symbol viewBox="0 0 24 24" id="jump-to">
<path d="M19 7v4H5.83l3.58-3.59L8 6l-6 6 6 6 1.41-1.41L5.83 13H21V7z"/>
</symbol>
<symbol viewBox="0 0 24 24" id="expand">
<path d="M10 18h4v-2h-4v2zM3 6v2h18V6H3zm3 7h12v-2H6v2z"/>
</symbol>
</defs>
</svg>
<div id="swagger-ui"></div>
<script src="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.2.1/swagger-ui-bundle.js"
crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.2.1/swagger-ui-standalone-preset.js"
crossorigin="anonymous"></script>
<script>
window.onload = function () {
const ui = SwaggerUIBundle({
url: window.location.href.replace(/\/swagger$/, "") + "/api/swagger.json",
dom_id: '#swagger-ui',
deepLinking: true,
presets: [
SwaggerUIBundle.presets.apis,
SwaggerUIStandalonePreset
],
plugins: [
SwaggerUIBundle.plugins.DownloadUrl
],
layout: "StandaloneLayout"
});
window.ui = ui
}
</script>
</body>
</html>
server/app/web/webapp.py
-7
View File
@@ -11,13 +11,6 @@ def index():
return render_template("index.html", datasetTitle=dataset_title)
# renders swagger documentation
@bp.route("/swagger")
def swag():
return render_template("swagger.html")
# renders swagger documentation
@bp.route("/favicon.png")
def favicon():
return send_from_directory(os.path.join(bp.root_path, "static/img/"), "favicon.png")