fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-10-06 09:58:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Flatbuffer cleanup (#598)

Browse Source 
* dead code and route removal

* more dead code cleanup

* fix scanpy_engine tests

* lint

* add missing catch in filter parsing

* update scanpy NaN tests

* more fbs tests and dead test removal

* remove forced default for content type negotiation

* bit of cleanup

* more fbs test cleanup

* lint

* remove swagger

* swagger cleanup

* lint

* correctly handle lack of templates

* more dead code removal

* remove unused files

* fix dev build

* lint
This commit is contained in:
Bruce Martin
2019-02-19 08:50:29 -08:00
committed by GitHub
parent 4e67c645f8
commit 57c4e9ff33
16 changed files with 210 additions and 1663 deletions
Download Patch File Download Diff File Expand all files Collapse all files
makefile
+3
View File
@@ -15,6 +15,7 @@ build-server : build-client
cp -r server/* $(SERVERBUILD) cp -r server/* $(SERVERBUILD)
cp -r client/build/ $(CLIENTBUILD) cp -r client/build/ $(CLIENTBUILD)
mkdir -p $(SERVERBUILD)/app/web/static/img mkdir -p $(SERVERBUILD)/app/web/static/img
mkdir -p $(SERVERBUILD)/app/web/templates/
cp $(CLIENTBUILD)/index.html $(SERVERBUILD)/app/web/templates/ cp $(CLIENTBUILD)/index.html $(SERVERBUILD)/app/web/templates/
cp -r $(CLIENTBUILD)/static $(SERVERBUILD)/app/web/ cp -r $(CLIENTBUILD)/static $(SERVERBUILD)/app/web/
cp $(CLIENTBUILD)/favicon.png $(SERVERBUILD)/app/web/static/img cp $(CLIENTBUILD)/favicon.png $(SERVERBUILD)/app/web/static/img
@@ -28,6 +29,8 @@ build-client :
# If you are actively developing in the server folder use this, dirties the source tree # If you are actively developing in the server folder use this, dirties the source tree
build-for-server-dev : clean-server build-client build-for-server-dev : clean-server build-client
mkdir -p server/app/web/static/img mkdir -p server/app/web/static/img
mkdir -p server/app/web/static/js
mkdir -p server/app/web/templates/
cp client/build/index.html server/app/web/templates/ cp client/build/index.html server/app/web/templates/
cp -r client/build/static server/app/web/ cp -r client/build/static server/app/web/
cp client/build/favicon.png server/app/web/static/img cp client/build/favicon.png server/app/web/static/img
server/app/app.py
-15
View File
@@ -4,7 +4,6 @@ from flask import Flask
from flask_caching import Cache from flask_caching import Cache
from flask_compress import Compress from flask_compress import Compress
from flask_cors import CORS from flask_cors import CORS
from flask_restful_swagger_2 import get_swagger_blueprint
from .rest_api.rest import get_api_resources from .rest_api.rest import get_api_resources
from .util.utils import Float32JSONEncoder from .util.utils import Float32JSONEncoder
@@ -26,21 +25,7 @@ app.config.update(SECRET_KEY=SECRET_KEY)
# Application Data # Application Data
data = None data = None
# A list of swagger document objects
docs = []
resources = get_api_resources() resources = get_api_resources()
docs.append(resources.get_swagger_doc())
app.register_blueprint(webapp.bp) app.register_blueprint(webapp.bp)
app.register_blueprint(resources.blueprint) app.register_blueprint(resources.blueprint)
app.register_blueprint(
get_swagger_blueprint(
docs,
"/api/swagger",
produces=["application/json"],
title="cellxgene rest api",
description="An API connecting ExpressionMatrix2 clustering algorithm to cellxgene",
)
)
app.add_url_rule("/", endpoint="index") app.add_url_rule("/", endpoint="index")
server/app/driver/driver.py
+2 -45
View File
@@ -42,45 +42,12 @@ class CXGDriver(metaclass=ABCMeta):
pass pass
@abstractmethod @abstractmethod
def filter_dataframe(self, filter): def annotation_to_fbs_matrix(self, axis, field=None):
"""
Filter cells from data and return a subset of the data. They can operate on both obs and var dimension with
indexing and filtering by annotation value. Filters are combined with the and operator.
See REST specs for info on filter format:
https://github.com/chanzuckerberg/cellxgene/blob/master/docs/REST_API.md
:param filter: dictionary with filter params
:return: View into scanpy object with cells/genes filtered
"""
pass
@abstractmethod
def annotation(self, filter, axis, fields=None):
""" """
Gets annotation value for each observation Gets annotation value for each observation
:param filter: filter: dictionary with filter params
:param axis: string obs or var :param axis: string obs or var
:param fields: list of keys for annotation to return, returns all annotation values if not set. :param fields: list of keys for annotation to return, returns all annotation values if not set.
:return: dict: names - list of fields in order, data - list of lists or metadata :return: flatbuffer: in fbs/matrix.fbs encoding
[observation ids, val1, val2...]
"""
pass
@abstractmethod
def annotation_to_fbs_matrix(self, axis, field=None):
""" Same as annotation(), except returns a flatbuffer, and does not support filtering. """
pass
@abstractmethod
def data_frame(self, filter, axis):
"""
Retrieves data for each variable for observations in data frame
:param filter: filter: dictionary with filter params
:param axis: string obs or var
:return: {
"var": list of variable ids,
"obs": [cellid, var1 expression, var2 expression, ...],
}
""" """
pass pass
@@ -104,16 +71,6 @@ class CXGDriver(metaclass=ABCMeta):
""" """
pass pass
@abstractmethod
def layout(self, filter, interactive_limit=None):
"""
Computes a n-d layout for cells through dimensionality reduction.
:param filter: filter: dictionary with filter params
:param interactive_limit: -- don't compute if total # genes in dataframes are larger than this
:return: [cellid, x, y, ...]
"""
pass
@abstractmethod @abstractmethod
def layout_to_fbs_matrix(self, filter): def layout_to_fbs_matrix(self, filter):
""" same as layout, except returns a flatbuffer """ """ same as layout, except returns a flatbuffer """
server/app/rest_api/rest.py
+13 -674
View File
@@ -3,22 +3,17 @@ import pkg_resources
import warnings import warnings
from flask import Blueprint, current_app, jsonify, make_response, request from flask import Blueprint, current_app, jsonify, make_response, request
from flask_restful_swagger_2 import Api, swagger, Resource from flask_restful import Api, Resource
from werkzeug.datastructures import ImmutableMultiDict
from server.app.util.constants import ( from server.app.util.constants import (
Axis, Axis,
DiffExpMode, DiffExpMode,
JSON_NaN_to_num_warning_msg, JSON_NaN_to_num_warning_msg,
) )
from server.app.util.filter import parse_filter, QueryStringError
from server.app.util.models import FilterModel
from server.app.util.utils import get_mime_type
from server.app.util.errors import ( from server.app.util.errors import (
FilterError, FilterError,
InteractiveError, InteractiveError,
JSONEncodingValueError, JSONEncodingValueError,
MimeTypeError,
PrepareError, PrepareError,
) )
@@ -31,47 +26,6 @@ Sort order for routes
class SchemaAPI(Resource): class SchemaAPI(Resource):
@swagger.doc(
{
"summary": "get schema for dataframe and annotations",
"tags": ["initialize"],
"parameters": [],
"responses": {
"200": {
"description": "schema",
"examples": {
"application/json": {
"schema": {
"dataframe": {
"nObs": 383,
"nVar": 19944,
"type": "float32",
},
"annotations": {
"obs": [
{"name": "name", "type": "string"},
{"name": "tissue_type", "type": "string"},
{"name": "num_reads", "type": "int32"},
{"name": "sample_name", "type": "string"},
{
"name": "clusters",
"type": "categorical",
"categories": [99, 1, "unknown cluster"],
},
{"name": "QScore", "type": "float32"},
],
"var": [
{"name": "name", "type": "string"},
{"name": "gene", "type": "string"},
],
},
}
}
},
}
},
}
)
def get(self): def get(self):
return make_response( return make_response(
jsonify({"schema": current_app.data.schema}), HTTPStatus.OK jsonify({"schema": current_app.data.schema}), HTTPStatus.OK
@@ -79,47 +33,6 @@ class SchemaAPI(Resource):
class ConfigAPI(Resource): class ConfigAPI(Resource):
@swagger.doc(
{
"summary": "Configuration information to assist in front-end adaptation"
" to underlying engine, available functionality, interactive time limits, etc",
"tags": ["initialize"],
"parameters": [],
"responses": {
"200": {
"description": "schema",
"examples": {
"application/json": {
"config": {
"features": [
{
"method": "POST",
"path": "/cluster/",
"available": False,
},
{
"method": "POST",
"path": "/layout/obs",
"available": True,
"interactiveLimit": 10000,
},
{
"method": "POST",
"path": "/layout/var",
"available": False,
},
],
"displayNames": {
"engine": "ScanPy version 1.33",
"dataset": "/home/joe/mouse/blorth.csv",
},
}
}
},
}
},
}
)
def get(self): def get(self):
config = { config = {
"config": { "config": {
@@ -158,51 +71,13 @@ class ConfigAPI(Resource):
class AnnotationsObsAPI(Resource): class AnnotationsObsAPI(Resource):
@swagger.doc(
{
"summary": "Fetch annotations (metadata) for all observations.",
"tags": ["annotations"],
"parameters": [
{
"in": "query",
"name": "annotation-name",
"type": "string",
"description": "list of 1 or more annotation names",
}
],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": ["tissue_type", "sex", "num_reads", "clusters"],
"data": [
[0, "lung", "F", 39844, 99],
[1, "heart", "M", 83, 1],
[49, "spleen", None, 2, "unknown cluster"],
],
}
},
},
"400": {
"description": "one or more of the annotation-name identifiers were not associated with an "
"annotation name"
},
},
}
)
def get(self): def get(self):
fields = request.args.getlist("annotation-name", None) fields = request.args.getlist("annotation-name", None)
preferred_mimetype = request.accept_mimetypes.best_match( preferred_mimetype = request.accept_mimetypes.best_match(
["application/json", "application/octet-stream"], ["application/octet-stream"]
"application/json"
) )
try: try:
if preferred_mimetype == "application/json": if preferred_mimetype == "application/octet-stream":
return make_response(
current_app.data.annotation({}, "obs", fields), HTTPStatus.OK, {"Content-Type": "application/json"}
)
elif preferred_mimetype == "application/octet-stream":
return make_response(current_app.data.annotation_to_fbs_matrix("obs", fields), return make_response(current_app.data.annotation_to_fbs_matrix("obs", fields),
HTTPStatus.OK, HTTPStatus.OK,
{"Content-Type": "application/octet-stream"}) {"Content-Type": "application/octet-stream"})
@@ -210,119 +85,18 @@ class AnnotationsObsAPI(Resource):
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
except KeyError: except KeyError:
return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST) return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
@swagger.doc(
{
"summary": "Fetch annotations (metadata) for filtered subset of observations.",
"tags": ["annotations"],
"parameters": [
{
"in": "query",
"name": "annotation-name",
"type": "string",
"description": "list of 1 or more annotation names",
},
{
"name": "filter",
"description": "Complex Filter",
"in": "body",
"schema": FilterModel,
},
],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": ["tissue_type", "sex", "num_reads", "clusters"],
"data": [
[0, "lung", "F", 39844, 99],
[1, "heart", "M", 83, 1],
[49, "spleen", None, 2, "unknown cluster"],
],
}
},
},
"400": {
"description": "malformed filter or one or more of the annotation-name identifiers were"
"not associated with an annotation name"
},
},
}
)
def put(self):
fields = request.args.getlist("annotation-name", None)
try:
annotation_response = current_app.data.annotation(
request.get_json()["filter"], "obs", fields
)
return make_response(
annotation_response, HTTPStatus.OK, {"Content-Type": "application/json"}
)
except KeyError:
return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e: except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
class AnnotationsVarAPI(Resource): class AnnotationsVarAPI(Resource):
@swagger.doc(
{
"summary": "Fetch annotations (metadata) for all variables.",
"tags": ["annotations"],
"parameters": [
{
"in": "query",
"name": "annotation-name",
"type": "string",
"description": "list of 1 or more annotation names",
}
],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": ["name", "category"],
"data": [
[0, "ATAD3C", 1],
[1, "RER1", None],
[49, "S100B", 6],
],
}
},
},
"400": {
"description": "one or more of the annotation-name identifiers were not associated with an"
" annotation name"
},
},
}
)
def get(self): def get(self):
fields = request.args.getlist("annotation-name", None) fields = request.args.getlist("annotation-name", None)
preferred_mimetype = request.accept_mimetypes.best_match( preferred_mimetype = request.accept_mimetypes.best_match(
["application/json", "application/octet-stream"], ["application/octet-stream"]
"application/json"
) )
try: try:
if preferred_mimetype == "application/json": if preferred_mimetype == "application/octet-stream":
return make_response(current_app.data.annotation({}, "var", fields),
HTTPStatus.OK,
{"Content-Type": "application/json"})
elif preferred_mimetype == "application/octet-stream":
return make_response(current_app.data.annotation_to_fbs_matrix("var", fields), return make_response(current_app.data.annotation_to_fbs_matrix("var", fields),
HTTPStatus.OK, HTTPStatus.OK,
{"Content-Type": "application/octet-stream"}) {"Content-Type": "application/octet-stream"})
@@ -330,317 +104,22 @@ class AnnotationsVarAPI(Resource):
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
except KeyError: except KeyError:
return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST) return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
@swagger.doc(
{
"summary": "Fetch annotations (metadata) for filtered subset of variables.",
"tags": ["annotations"],
"parameters": [
{
"in": "query",
"name": "annotation-name",
"type": "string",
"description": "list of 1 or more annotation names",
},
{
"name": "filter",
"description": "Complex Filter",
"in": "body",
"schema": FilterModel,
},
],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": ["name", "category"],
"data": [
[0, "ATAD3C", 1],
[1, "RER1", None],
[49, "S100B", 6],
],
}
},
},
"400": {
"description": "malformed filter or one or more of the annotation-name identifiers were"
"not associated with an annotation name"
},
},
}
)
def put(self):
fields = request.args.getlist("annotation-name", None)
try:
annotation_response = current_app.data.annotation(
request.get_json()["filter"], "var", fields
)
return make_response(
annotation_response, HTTPStatus.OK, {"Content-Type": "application/json"}
)
except KeyError:
return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
except FilterError:
return make_response("Malformed filter", HTTPStatus.BAD_REQUEST)
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
class DataObsAPI(Resource):
@swagger.doc(
{
"summary": "Get data (expression values) from the dataframe.",
"tags": ["data"],
"parameters": [
{
"in": "query",
"name": "filter",
"type": "string",
"description": "axis:key:value",
},
{
"in": "query",
"name": "accept-type",
"type": "string",
"description": "MIME type",
},
],
"responses": {
"200": {
"description": "expression",
"examples": {
"application/json": {
"var": [0, 20000],
"obs": [[1, 39483, 3902, 203, 0, 0, 28]],
}
},
},
"400": {"description": "Malformed filter"},
"406": {"description": "Unacceptable MIME type"},
},
}
)
def get(self):
accept_type = request.args.get("accept-type", None)
# request.args is immutable
args = request.args.copy()
args.pop("accept-type", None)
try:
filter_ = parse_filter(
ImmutableMultiDict(args), current_app.data.schema["annotations"]
)
except QueryStringError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
# TODO support CSV
try:
# TODO store mime_type when more than one is supported
get_mime_type(
acceptable_types=["application/json"],
query_param=accept_type,
header=request.accept_mimetypes,
)
except MimeTypeError as e:
return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
try:
return make_response(
current_app.data.data_frame(filter_, axis=Axis.OBS),
HTTPStatus.OK,
{"Content-Type": "application/json"},
)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
@swagger.doc(
{
"summary": "Get data (expression values) from the dataframe.",
"tags": ["data"],
"parameters": [
{
"name": "filter",
"description": "Complex Filter",
"in": "body",
"schema": FilterModel,
}
],
"responses": {
"200": {
"description": "expression",
"examples": {
"application/json": {
"var": [0, 20000],
"obs": [[1, 39483, 3902, 203, 0, 0, 28]],
}
},
},
"400": {"description": "Malformed filter"},
"406": {"description": "Unacceptable MIME type"},
},
}
)
def put(self):
if not request.accept_mimetypes.best_match(["application/json", "text/csv"]):
return make_response(
f"Unsupported MIME type '{request.accept_mimetypes}'",
HTTPStatus.NOT_ACCEPTABLE,
)
try:
get_mime_type(
acceptable_types=["application/json"], header=request.accept_mimetypes
)
except MimeTypeError as e:
return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
try:
return make_response(
(
current_app.data.data_frame(
request.get_json()["filter"], axis=Axis.OBS
)
),
HTTPStatus.OK,
{"Content-Type": "application/json"},
)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e: except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
class DataVarAPI(Resource): class DataVarAPI(Resource):
@swagger.doc(
{
"summary": "Get data (expression values) from the dataframe.",
"tags": ["data"],
"parameters": [
{
"in": "query",
"name": "filter",
"type": "string",
"description": "axis:key:value",
},
{
"in": "query",
"name": "accept-type",
"type": "string",
"description": "MIME type",
},
],
"responses": {
"200": {
"description": "expression",
"examples": {
"application/json": {
"obs": [0, 20000],
"var": [[1, 39483, 3902, 203, 0, 0, 28]],
}
},
},
"400": {"description": "Malformed filter"},
"406": {"description": "Unacceptable MIME type"},
},
}
)
def get(self):
accept_type = request.args.get("accept-type", None)
# request.args is immutable
args = request.args.copy()
args.pop("accept-type", None)
try:
filter_ = parse_filter(
ImmutableMultiDict(args), current_app.data.schema["annotations"]
)
except QueryStringError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
try:
get_mime_type(
acceptable_types=["application/json"],
query_param=accept_type,
header=request.accept_mimetypes,
)
except MimeTypeError as e:
return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
try:
return make_response(
current_app.data.data_frame(filter_, axis=Axis.VAR),
HTTPStatus.OK,
{"Content-Type": "application/json"},
)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
@swagger.doc(
{
"summary": "Get data (expression values) from the dataframe.",
"tags": ["data"],
"parameters": [
{
"name": "filter",
"description": "Complex Filter",
"in": "body",
"schema": FilterModel,
}
],
"responses": {
"200": {
"description": "expression",
"examples": {
"application/json": {
"obs": [0, 20000],
"var": [[1, 39483, 3902, 203, 0, 0, 28]],
}
},
},
"400": {"description": "Malformed filter"},
"406": {"description": "Unacceptable MIME type"},
},
}
)
def put(self): def put(self):
preferred_mimetype = request.accept_mimetypes.best_match( preferred_mimetype = request.accept_mimetypes.best_match(
["application/json", "application/octet-stream"], ["application/octet-stream"]
"application/json"
) )
try: try:
if preferred_mimetype == "application/json": if preferred_mimetype == "application/octet-stream":
return make_response( filter_json = request.get_json()
( filter = filter_json["filter"] if filter_json else None
current_app.data.data_frame(
request.get_json()["filter"], axis=Axis.VAR
)
),
HTTPStatus.OK,
{"Content-Type": "application/json"},
)
elif preferred_mimetype == "application/octet-stream":
return make_response( return make_response(
current_app.data.data_frame_to_fbs_matrix( current_app.data.data_frame_to_fbs_matrix(
request.get_json()["filter"], axis=Axis.VAR filter, axis=Axis.VAR
), ),
HTTPStatus.OK, HTTPStatus.OK,
{"Content-Type": "application/octet-stream"}) {"Content-Type": "application/octet-stream"})
@@ -648,73 +127,11 @@ class DataVarAPI(Resource):
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
except FilterError as e: except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST) return make_response(e.message, HTTPStatus.BAD_REQUEST)
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e: except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
class DiffExpObsAPI(Resource): class DiffExpObsAPI(Resource):
@swagger.doc(
{
"summary": "Generate differential expression (DE) statistics for two specified subsets of data, "
"as indicated by the two provided observation complex filters",
"tags": ["diffexp"],
# TODO sort out params
# "parameters": [
# # {
# # "in": "body",
# # "name": "mode",
# # "type": "string",
# # "required": True,
# # "description": "topN or varFilter"
# # },
# {
# "in": "query",
# "name": "count",
# "type": "int32",
# "description": "TopN mode: how many vars to return"
# },
# {
# "in": "body",
# "name": "varFilter",
# "schema": FilterModel,
# "description": "varFilter: Complex filter, only var for which vars to return"
# },
# {
# "in": "body",
# "name": "set1",
# "schema": FilterModel,
# "required": True,
# "description": "Complex filter, only obs - observations in set1"
# },
# {
# "in": "body",
# "name": "set2",
# "schema": FilterModel,
# "description": "Complex filter, only obs - observations in set2.
# If not included, inverse of set1."
# },
# ],
"responses": {
"200": {
"description": "Statistics are encoded as an array of arrays, with fields ordered as: "
"varIndex, logfoldchange, pVal, pValAdj",
"examples": {
"application/json": [
[328, -2.569_489, 2.655_706e-63, 3.642_036e-57],
[1250, -2.569_489, 2.655_706e-63, 3.642_036e-57],
]
},
},
"400": {"description": "malformed filter"},
"403": {"description": "non-interactive request"},
"501": {"description": "diffexp is not implemented"},
},
}
)
def post(self): def post(self):
args = request.get_json() args = request.get_json()
# confirm mode is present and legal # confirm mode is present and legal
@@ -783,40 +200,12 @@ class DiffExpObsAPI(Resource):
class LayoutObsAPI(Resource): class LayoutObsAPI(Resource):
@swagger.doc(
{
"summary": "Get the default layout for all observations.",
"tags": ["layout"],
"parameters": [],
"responses": {
"200": {
"description": "layout",
"examples": {
"application/json": {
"layout": {
"ndims": 2,
"coordinates": [
[0, 0.284_483, 0.983_744],
[1, 0.038_844, 0.739_444],
],
}
}
},
},
"400": {"description": "Data preparation error"},
},
}
)
def get(self): def get(self):
preferred_mimetype = request.accept_mimetypes.best_match( preferred_mimetype = request.accept_mimetypes.best_match(
["application/json", "application/octet-stream"], ["application/octet-stream"]
"application/json"
) )
try: try:
if preferred_mimetype == "application/json": if preferred_mimetype == "application/octet-stream":
return make_response(current_app.data.layout({}), HTTPStatus.OK, {"Content-Type": "application/json"})
elif preferred_mimetype == "application/octet-stream":
return make_response(current_app.data.layout_to_fbs_matrix(), return make_response(current_app.data.layout_to_fbs_matrix(),
HTTPStatus.OK, HTTPStatus.OK,
{"Content-Type": "application/octet-stream"}) {"Content-Type": "application/octet-stream"})
@@ -824,69 +213,19 @@ class LayoutObsAPI(Resource):
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
except PrepareError as e: except PrepareError as e:
return make_response(e.message, HTTPStatus.INTERNAL_SERVER_ERROR) return make_response(e.message, HTTPStatus.INTERNAL_SERVER_ERROR)
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e: except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
# @swagger.doc({
# "summary": "Observation layout for filtered subset.",
# "tags": ["layout"],
# "parameters": [
# {
# "name": "filter",
# "description": "Complex Filter",
# "in": "body",
# "schema": FilterModel
# }
# ],
# "responses": {
# "200": {
# "description": "layout",
# "examples": {
# "application/json": {
# "layout": {
# "ndims": 2,
# "coordinates": [
# [0, 0.284483, 0.983744],
# [1, 0.038844, 0.739444]
# ]
# }
# }
# }
# },
# "400": {
# "description": "Malformed filter"
# },
# "403": {
# "description": "Non-interactive request"
# },
# }
# })
# def put(self):
# try:
# filter = request.get_json()["filter"]
# interactive_limit = current_app.data.features["layout"]["obs"]["interactiveLimit"]
# layout = current_app.data.layout(filter, interactive_limit=interactive_limit)
# return make_response(layout, HTTPStatus.OK, {"Content-Type": content_type})
# except FilterError as e:
# return make_response(e.message, HTTPStatus.BAD_REQUEST)
# except InteractiveError:
# return make_response("Non-interactive request", HTTPStatus.FORBIDDEN)
def get_api_resources(): def get_api_resources():
bp = Blueprint("api", __name__, url_prefix="/api/v0.2") bp = Blueprint("api", __name__, url_prefix="/api/v0.2")
api = Api(bp, add_api_spec_resource=False) api = Api(bp)
# Initialization routes # Initialization routes
api.add_resource(SchemaAPI, "/schema") api.add_resource(SchemaAPI, "/schema")
api.add_resource(ConfigAPI, "/config") api.add_resource(ConfigAPI, "/config")
# Data routes # Data routes
api.add_resource(AnnotationsObsAPI, "/annotations/obs") api.add_resource(AnnotationsObsAPI, "/annotations/obs")
api.add_resource(AnnotationsVarAPI, "/annotations/var") api.add_resource(AnnotationsVarAPI, "/annotations/var")
api.add_resource(DataObsAPI, "/data/obs")
api.add_resource(DataVarAPI, "/data/var") api.add_resource(DataVarAPI, "/data/var")
# Computation routes # Computation routes
api.add_resource(DiffExpObsAPI, "/diffexp/obs") api.add_resource(DiffExpObsAPI, "/diffexp/obs")
server/app/scanpy_engine/scanpy_engine.py
+1 -169
View File
@@ -1,16 +1,13 @@
import warnings import warnings
import numpy as np import numpy as np
from pandas import DataFrame
from pandas.core.dtypes.dtypes import CategoricalDtype from pandas.core.dtypes.dtypes import CategoricalDtype
import scanpy.api as sc import scanpy.api as sc
from scipy import sparse
from server.app.driver.driver import CXGDriver from server.app.driver.driver import CXGDriver
from server.app.util.constants import Axis, DEFAULT_TOP_N from server.app.util.constants import Axis, DEFAULT_TOP_N
from server.app.util.errors import ( from server.app.util.errors import (
FilterError, FilterError,
InteractiveError,
JSONEncodingValueError, JSONEncodingValueError,
PrepareError, PrepareError,
ScanpyFileError, ScanpyFileError,
@@ -197,23 +194,6 @@ class ScanpyEngine(CXGDriver):
f"to solve this problem. " f"to solve this problem. "
) )
def filter_dataframe(self, filter):
"""
Filter cells from data and return a subset of the data. They can operate on both obs and var dimension with
indexing and filtering by annotation value. Filters are combined with the and operator.
See REST specs for info on filter format:
# TODO update this link to swagger when it's done
https://docs.google.com/document/d/1Fxjp1SKtCk7l8QP9-7KAjGXL0eldi_qEnNT0NmlGzXI/edit#heading=h.8qc9q57amldx
:param filter: dictionary with filter params
:return: View into scanpy object with cells/genes filtered
"""
if not filter:
return self.data
obs_selector, var_selector = self._filter_to_mask(filter, use_slices=False)
data = self._slice(self.data, obs_selector, var_selector)
return data
@staticmethod @staticmethod
def _annotation_filter_to_mask(filter, d_axis, count): def _annotation_filter_to_mask(filter, d_axis, count):
mask = np.ones((count,), dtype=bool) mask = np.ones((count,), dtype=bool)
@@ -277,72 +257,6 @@ class ScanpyEngine(CXGDriver):
) )
return obs_selector, var_selector return obs_selector, var_selector
@staticmethod
def _slice(data, obs_selector=None, vars_selector=None):
"""
Slice date using any selector that the AnnData object
supprots for slicing. If selector is None, will not slice
on that axis.
This method exists to optimize filtering/slicing sparse data that has
access patterns which impact slicing performance.
https://docs.scipy.org/doc/scipy/reference/sparse.html
"""
prefer_row_access = (
sparse.isspmatrix_csr(data._X)
or sparse.isspmatrix_lil(data._X)
or sparse.isspmatrix_bsr(data._X)
)
if prefer_row_access:
# Row-major slicing
if obs_selector is not None:
data = data[obs_selector, :]
if vars_selector is not None:
data = data[:, vars_selector]
else:
# Col-major slicing
if vars_selector is not None:
data = data[:, vars_selector]
if obs_selector is not None:
data = data[obs_selector, :]
return data
def annotation(self, filter, axis, fields=None):
"""
Gets annotation value for each observation
:param filter: filter: dictionary with filter params
:param axis: string obs or var
:param fields: list of keys for annotation to return, returns all annotation values if not set.
:return: dict: names - list of fields in order, data - list of lists or metadata
[observation ids, val1, val2...]
"""
try:
obs_selector, var_selector = self._filter_to_mask(filter)
except (KeyError, IndexError) as e:
raise FilterError(f"Error parsing filter: {e}") from e
if axis == Axis.OBS:
obs = self.data.obs[obs_selector]
if not fields:
fields = obs.columns.tolist()
result = {
"names": fields,
"data": DataFrame(obs[fields]).to_records(index=True).tolist(),
}
else:
var = self.data.var[var_selector]
if not fields:
fields = var.columns.tolist()
result = {
"names": fields,
"data": DataFrame(var[fields]).to_records(index=True).tolist(),
}
try:
return jsonify_scanpy(result)
except ValueError:
raise JSONEncodingValueError("Error encoding annotations to JSON")
def annotation_to_fbs_matrix(self, axis, fields=None): def annotation_to_fbs_matrix(self, axis, fields=None):
if axis == Axis.OBS: if axis == Axis.OBS:
df = self.data.obs df = self.data.obs
@@ -352,44 +266,6 @@ class ScanpyEngine(CXGDriver):
df = df[fields] df = df[fields]
return encode_matrix_fbs(df, col_idx=df.columns) return encode_matrix_fbs(df, col_idx=df.columns)
def data_frame(self, filter, axis):
"""
Retrieves data for each variable for observations in data frame
:param filter: filter: dictionary with filter params
:param axis: string obs or var
:return: {
"var": list of variable ids,
"obs": [cellid, var1 expression, var2 expression, ...],
}
"""
try:
obs_selector, var_selector = self._filter_to_mask(filter)
except (KeyError, IndexError) as e:
raise FilterError(f"Error parsing filter: {e}") from e
_X = self.data._X[obs_selector, var_selector]
if sparse.issparse(_X):
_X = _X.toarray()
var_index_sliced = self.data.var.index[var_selector]
obs_index_sliced = self.data.obs.index[obs_selector]
if axis == Axis.OBS:
result = {
"var": var_index_sliced.tolist(),
"obs": DataFrame(_X, index=obs_index_sliced)
.to_records(index=True)
.tolist(),
}
else:
result = {
"obs": obs_index_sliced.tolist(),
"var": DataFrame(_X.T, index=var_index_sliced)
.to_records(index=True)
.tolist(),
}
try:
return jsonify_scanpy(result)
except ValueError:
raise JSONEncodingValueError("Error encoding dataframe to JSON")
def data_frame_to_fbs_matrix(self, filter, axis): def data_frame_to_fbs_matrix(self, filter, axis):
""" """
Retrieves data 'X' and returns in a flatbuffer Matrix. Retrieves data 'X' and returns in a flatbuffer Matrix.
@@ -405,7 +281,7 @@ class ScanpyEngine(CXGDriver):
raise ValueError("Only VAR dimension access is supported") raise ValueError("Only VAR dimension access is supported")
try: try:
obs_selector, var_selector = self._filter_to_mask(filter, use_slices=False) obs_selector, var_selector = self._filter_to_mask(filter, use_slices=False)
except (KeyError, IndexError) as e: except (KeyError, IndexError, TypeError) as e:
raise FilterError(f"Error parsing filter: {e}") from e raise FilterError(f"Error parsing filter: {e}") from e
if obs_selector is not None: if obs_selector is not None:
raise FilterError("filtering on obs unsupported") raise FilterError("filtering on obs unsupported")
@@ -440,50 +316,6 @@ class ScanpyEngine(CXGDriver):
"Error encoding differential expression to JSON" "Error encoding differential expression to JSON"
) )
def layout(self, filter, interactive_limit=None):
"""
Computes a n-d layout for cells through dimensionality reduction.
:param filter: filter: dictionary with filter params
:param interactive_limit: -- don't compute if total # genes in dataframes are larger than this
:return: [cellid, x, y, ...]
"""
try:
df = self.filter_dataframe(filter)
except (KeyError, IndexError) as e:
raise FilterError(f"Error parsing filter: {e}") from e
if interactive_limit and len(df.obs.index) > interactive_limit:
raise InteractiveError("Size data is too large for interactive computation")
# TODO Filtering cells is fine, but filtering genes does nothing because the neighbors are
# calculated using the original vars (geneset) and this doesn’t get updated when you use less.
# Need to recalculate neighbors (long) if user requests new layout filtered by var
# TODO for MVP we are pushing computation of layout to preprocessing and not allowing re-layout
# this will probably change after user feedback
# getattr(sc.tl, self.layout_method)(df, random_state=123)
try:
df_layout = df.obsm[f"X_{self.layout_method}"]
except ValueError as e:
raise PrepareError(
f"Layout has not been calculated using {self.layout_method}, "
f"please prepare your datafile and relaunch cellxgene"
) from e
normalized_layout = DataFrame(
(df_layout - df_layout.min()) / (df_layout.max() - df_layout.min()),
index=df.obs.index,
)
try:
return jsonify_scanpy(
{
"layout": {
"ndims": normalized_layout.shape[1],
"coordinates": normalized_layout.to_records(
index=True
).tolist(),
}
}
)
except ValueError:
raise JSONEncodingValueError("Error encoding layout to JSON")
def layout_to_fbs_matrix(self): def layout_to_fbs_matrix(self):
""" """
Return the default 2-D layout for cells as a FBS Matrix. Return the default 2-D layout for cells as a FBS Matrix.
server/app/util/filter.py
-86
View File
@@ -1,86 +0,0 @@
import json
from collections import defaultdict
from numpy import float32, int32
from server.app.util.constants import Axis
class QueryStringError(Exception):
def __init__(self, key, message):
self.key = key
self.message = message
def _convert_variable(datatype, variable):
"""
Convert variable to number (float/int)
Used for dataset metadata and for query string
:param datatype: type to convert to
:param variable (string or None): value of variable
:return: converted variable
:raises: AssertionError
"""
assert datatype in ["boolean", "categorical", "float32", "int32", "string"]
if variable is None:
return variable
if datatype == "int32":
variable = int32(variable)
elif datatype == "float32":
variable = float32(variable)
elif datatype == "boolean":
variable = json.loads(variable)
assert isinstance(variable, bool)
return variable
def parse_filter(query_filter, schema):
"""
The filter comes in as arguments from a GET request
For categorical metadata keys filter based on axis:key=value
For continuous metadata keys filter by axis:key=min,max
Either value can be replaced by a * To have only a minimum
value axis:key=min,* To have only a maximum value axis:key=*,max
They combine via AND so a cell's metadata would have to match every filter
The results is a matrix with the cells the pass the filter and at this point all the genes
:param query_filter: flask's request.args
:param schema: dictionary schema
:raises QueryStringError
:return:
"""
query = defaultdict(lambda: defaultdict(list))
for key in query_filter:
axis, annotation = key.split(":", 1)
try:
Axis(axis)
except ValueError:
raise QueryStringError(key, f"Error: key {key} not in metadata schema")
ann_filter = {"name": annotation}
for ann in schema[axis]:
if ann["name"] == annotation:
dtype = ann["type"]
break
else:
raise QueryStringError(key, f"Error: {annotation} not a valid annotation name")
if dtype in ["string", "categorical", "boolean"]:
ann_filter["values"] = [_convert_variable(dtype, i) for i in query_filter.getlist(key)]
else:
value = query_filter.get(key)
try:
min_, max_ = value.split(",")
except ValueError:
raise QueryStringError(key, f"Error: min,max format required for range for {annotation}, got {value}")
if min_ == "*":
min_ = None
if max_ == "*":
max_ = None
try:
ann_filter["min"] = _convert_variable(dtype, min_)
ann_filter["max"] = _convert_variable(dtype, max_)
except ValueError:
raise QueryStringError(key, f"Error: expected type {query[key]['type']} for key {key}, got {value}")
query[axis]["annotation_value"].append(ann_filter)
return query
server/app/util/models.py
-34
View File
@@ -1,34 +0,0 @@
from flask_restful_swagger_2 import Schema
class AnnotationModel(Schema):
type = "object"
description = "Filter by annotation key: value"
properties = {
"name": {"type": "string"},
# TODO update to OpenAPI v3.0 when a library is available that supports it
# Unfortunately 2.0 doesn't have a way to have a schema that accepts multiple types
# Overloading the type key with a list seems to work ok and makes it to the page
"values": {"type": "array", "items": {"type": ["float32", "string", "int32", "bool"]}},
"min": {"type": ["int32", "float32"]},
"max": {"type": ["int32", "float32"]},
}
required = ["name"]
class IndexModel(Schema):
type = "object"
description = "Filter by index of observation/variable ex. [0, 5, 15]"
properties = {"index": {"type": "array", "items": {"format": "int32", "type": "integer"}}}
class AxisModel(Schema):
type = "object"
description = "Axis of data -- obs or var"
properties = {"index": IndexModel, "annotation_value": AnnotationModel.array()}
class FilterModel(Schema):
type = "object"
description = "Complex filter"
properties = {"filter": {"type": "object", "properties": {"obs": AxisModel, "var": AxisModel}}}
server/app/util/utils.py
-30
View File
@@ -1,10 +1,6 @@
import json import json
from argparse import ArgumentTypeError
from numpy import float32, integer from numpy import float32, integer
from server.app.util.errors import MimeTypeError
class Float32JSONEncoder(json.JSONEncoder): class Float32JSONEncoder(json.JSONEncoder):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
@@ -30,31 +26,5 @@ def custom_format_warning(msg, *args, **kwargs):
return f"[cellxgene] Warning: {msg} \n" return f"[cellxgene] Warning: {msg} \n"
def get_mime_type(
default="application/json", acceptable_types=["application/json", "text/csv"], query_param=None, header=None
):
mime_type = default
if query_param:
if query_param in acceptable_types:
mime_type = query_param
else:
raise MimeTypeError(f"Unsupported mime type {query_param} specified in query parameter 'accept-type'")
elif len(header):
mime_type = header.best_match(acceptable_types)
if not mime_type:
raise MimeTypeError(f"Unsupported mime type(s) {header} in HTTP Accept header")
return mime_type
def whole_number(value):
try:
value = int(value)
except ValueError as e:
raise ArgumentTypeError(f"{value} is not type int") from e
if value < 0:
raise ArgumentTypeError(f"{value} is not >= 0")
return value
def jsonify_scanpy(data): def jsonify_scanpy(data):
return json.dumps(data, cls=Float32JSONEncoder, allow_nan=False) return json.dumps(data, cls=Float32JSONEncoder, allow_nan=False)
server/app/web/templates/swagger.html
-97
View File
@@ -1,97 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>cellxgene REST API - Swagger definition</title>
<link href="https://fonts.googleapis.com/css?family=Open+Sans:400,700|Source+Code+Pro:300,600|Titillium+Web:400,600,700"
rel="stylesheet">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.2.1/swagger-ui.css"
crossorigin="anonymous"/>
<style>
html {
box-sizing: border-box;
overflow: -moz-scrollbars-vertical;
overflow-y: scroll;
}
*,
*:before,
*:after {
box-sizing: inherit;
}
body {
margin: 0;
background: #fafafa;
}
</style>
</head>
<body>
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"
style="position:absolute;width:0;height:0">
<defs>
<symbol viewBox="0 0 20 20" id="unlocked">
<path d="M15.8 8H14V5.6C14 2.703 12.665 1 10 1 7.334 1 6 2.703 6 5.6V6h2v-.801C8 3.754 8.797 3 10 3c1.203 0 2 .754 2 2.199V8H4c-.553 0-1 .646-1 1.199V17c0 .549.428 1.139.951 1.307l1.197.387C5.672 18.861 6.55 19 7.1 19h5.8c.549 0 1.428-.139 1.951-.307l1.196-.387c.524-.167.953-.757.953-1.306V9.199C17 8.646 16.352 8 15.8 8z"></path>
</symbol>
<symbol viewBox="0 0 20 20" id="locked">
<path d="M15.8 8H14V5.6C14 2.703 12.665 1 10 1 7.334 1 6 2.703 6 5.6V8H4c-.553 0-1 .646-1 1.199V17c0 .549.428 1.139.951 1.307l1.197.387C5.672 18.861 6.55 19 7.1 19h5.8c.549 0 1.428-.139 1.951-.307l1.196-.387c.524-.167.953-.757.953-1.306V9.199C17 8.646 16.352 8 15.8 8zM12 8H8V5.199C8 3.754 8.797 3 10 3c1.203 0 2 .754 2 2.199V8z"/>
</symbol>
<symbol viewBox="0 0 20 20" id="close">
<path d="M14.348 14.849c-.469.469-1.229.469-1.697 0L10 11.819l-2.651 3.029c-.469.469-1.229.469-1.697 0-.469-.469-.469-1.229 0-1.697l2.758-3.15-2.759-3.152c-.469-.469-.469-1.228 0-1.697.469-.469 1.228-.469 1.697 0L10 8.183l2.651-3.031c.469-.469 1.228-.469 1.697 0 .469.469.469 1.229 0 1.697l-2.758 3.152 2.758 3.15c.469.469.469 1.229 0 1.698z"/>
</symbol>
<symbol viewBox="0 0 20 20" id="large-arrow">
<path d="M13.25 10L6.109 2.58c-.268-.27-.268-.707 0-.979.268-.27.701-.27.969 0l7.83 7.908c.268.271.268.709 0 .979l-7.83 7.908c-.268.271-.701.27-.969 0-.268-.269-.268-.707 0-.979L13.25 10z"/>
</symbol>
<symbol viewBox="0 0 20 20" id="large-arrow-down">
<path d="M17.418 6.109c.272-.268.709-.268.979 0s.271.701 0 .969l-7.908 7.83c-.27.268-.707.268-.979 0l-7.908-7.83c-.27-.268-.27-.701 0-.969.271-.268.709-.268.979 0L10 13.25l7.418-7.141z"/>
</symbol>
<symbol viewBox="0 0 24 24" id="jump-to">
<path d="M19 7v4H5.83l3.58-3.59L8 6l-6 6 6 6 1.41-1.41L5.83 13H21V7z"/>
</symbol>
<symbol viewBox="0 0 24 24" id="expand">
<path d="M10 18h4v-2h-4v2zM3 6v2h18V6H3zm3 7h12v-2H6v2z"/>
</symbol>
</defs>
</svg>
<div id="swagger-ui"></div>
<script src="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.2.1/swagger-ui-bundle.js"
crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.2.1/swagger-ui-standalone-preset.js"
crossorigin="anonymous"></script>
<script>
window.onload = function () {
const ui = SwaggerUIBundle({
url: window.location.href.replace(/\/swagger$/, "") + "/api/swagger.json",
dom_id: '#swagger-ui',
deepLinking: true,
presets: [
SwaggerUIBundle.presets.apis,
SwaggerUIStandalonePreset
],
plugins: [
SwaggerUIBundle.plugins.DownloadUrl
],
layout: "StandaloneLayout"
});
window.ui = ui
}
</script>
</body>
</html>
server/app/web/webapp.py
-7
View File
@@ -11,13 +11,6 @@ def index():
return render_template("index.html", datasetTitle=dataset_title) return render_template("index.html", datasetTitle=dataset_title)
# renders swagger documentation
@bp.route("/swagger")
def swag():
return render_template("swagger.html")
# renders swagger documentation
@bp.route("/favicon.png") @bp.route("/favicon.png")
def favicon(): def favicon():
return send_from_directory(os.path.join(bp.root_path, "static/img/"), "favicon.png") return send_from_directory(os.path.join(bp.root_path, "static/img/"), "favicon.png")
server/requirements.txt
-1
View File
@@ -5,7 +5,6 @@ Flask-Caching>=1.4.0
Flask-Compress>=1.4.0 Flask-Compress>=1.4.0
Flask-Cors>=3.0.6 Flask-Cors>=3.0.6
Flask-RESTful>=0.3.6 Flask-RESTful>=0.3.6
flask-restful-swagger-2>=0.35
flatbuffers>=1.10.0 flatbuffers>=1.10.0
matplotlib>=2.2 matplotlib>=2.2
numpy>=1.14.5 numpy>=1.14.5
server/test/test_api.py
+71 -292
View File
@@ -57,16 +57,6 @@ class EndPoints(unittest.TestCase):
self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k") self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k")
self.assertEqual(len(result_data["config"]["features"]), 4) self.assertEqual(len(result_data["config"]["features"]), 4)
def test_get_layout(self):
endpoint = "layout/obs"
url = f"{URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["layout"]["ndims"], 2)
self.assertEqual(len(result_data["layout"]["coordinates"]), 2638)
def test_get_layout_fbs(self): def test_get_layout_fbs(self):
endpoint = "layout/obs" endpoint = "layout/obs"
url = f"{URL_BASE}{endpoint}" url = f"{URL_BASE}{endpoint}"
@@ -82,53 +72,11 @@ class EndPoints(unittest.TestCase):
self.assertIsNone(df['row_idx']) self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols']) self.assertEqual(len(df['columns']), df['n_cols'])
# def test_put_layout(self):
# endpoint = "layout/obs"
# url = f"{URL_BASE}{endpoint}"
# obs_filter = {
# "filter": {
# "obs": {
# "annotation_value": [
# {"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
# {"name": "n_counts", "min": 3000},
# ],
# "index": [1, 99, [1000, 2000]]
# }
# }
# }
# result = self.session.put(url, json=obs_filter)
# self.assertEqual(result.status_code, HTTPStatus.OK)
# result_data = result.json()
# self.assertEqual(len(result_data["layout"]["coordinates"]), 15)
def test_bad_filter(self): def test_bad_filter(self):
endpoints = ["annotations/obs", "annotations/var", "data/obs", "data/var"] endpoint = "data/var"
for endpoint in endpoints:
url = f"{URL_BASE}{endpoint}"
result = self.session.put(url, json=BAD_FILTER)
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
def test_get_annotations_obs(self):
endpoint = "annotations/obs"
url = f"{URL_BASE}{endpoint}" url = f"{URL_BASE}{endpoint}"
result = self.session.get(url) result = self.session.put(url, json=BAD_FILTER)
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["names"], ["name", "n_genes", "percent_mito", "n_counts", "louvain"])
self.assertEqual(len(result_data["data"]), 2638)
self.assertEqual(len(result_data["data"][0]), 6)
def test_get_annotations_obs_keys(self):
endpoint = "annotations/obs"
query = "annotation-name=n_genes&annotation-name=percent_mito"
url = f"{URL_BASE}{endpoint}?{query}"
result = self.session.get(url)
self.assertEqual(result.headers["Content-Type"], "application/json")
self.assertEqual(result.status_code, HTTPStatus.OK)
result_data = result.json()
self.assertEqual(result_data["names"], ["n_genes", "percent_mito"])
self.assertEqual(len(result_data["data"][0]), 3)
def test_get_annotations_obs_fbs(self): def test_get_annotations_obs_fbs(self):
endpoint = "annotations/obs" endpoint = "annotations/obs"
@@ -146,6 +94,23 @@ class EndPoints(unittest.TestCase):
self.assertEqual(len(df['columns']), df['n_cols']) self.assertEqual(len(df['columns']), df['n_cols'])
self.assertListEqual(df['col_idx'], ['name', 'n_genes', 'percent_mito', 'n_counts', 'louvain']) self.assertListEqual(df['col_idx'], ['name', 'n_genes', 'percent_mito', 'n_counts', 'louvain'])
def test_get_annotations_obs_keys_fbs(self):
endpoint = "annotations/obs"
query = "annotation-name=n_genes&annotation-name=percent_mito"
url = f"{URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 2)
self.assertIsNotNone(df['columns'])
self.assertIsNotNone(df['col_idx'])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
self.assertListEqual(df['col_idx'], ['n_genes', 'percent_mito'])
def test_get_annotations_obs_error(self): def test_get_annotations_obs_error(self):
endpoint = "annotations/obs" endpoint = "annotations/obs"
query = "annotation-name=notakey" query = "annotation-name=notakey"
@@ -153,50 +118,6 @@ class EndPoints(unittest.TestCase):
result = self.session.get(url) result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST) self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
def test_put_annotations_obs(self):
endpoint = "annotations/obs"
url = f"{URL_BASE}{endpoint}"
obs_filter = {
"filter": {
"obs": {
"annotation_value": [
{"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
{"name": "n_counts", "min": 3000},
],
"index": [1, 99, [1000, 2000]],
}
}
}
result = self.session.put(url, json=obs_filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["names"], ["name", "n_genes", "percent_mito", "n_counts", "louvain"])
self.assertEqual(len(result_data["data"]), 15)
def test_filter_put_annotations_obs(self):
endpoint = "annotations/obs"
query = "annotation-name=n_genes&annotation-name=percent_mito"
url = f"{URL_BASE}{endpoint}?{query}"
obs_filter = {
"filter": {
"obs": {
"annotation_value": [
{"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
{"name": "n_counts", "min": 3000},
],
"index": [1, 99, [1000, 2000]],
}
}
}
result = self.session.put(url, json=obs_filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["names"], ["n_genes", "percent_mito"])
self.assertEqual(len(result_data["data"][0]), 3)
self.assertEqual(len(result_data["data"]), 15)
def test_diff_exp(self): def test_diff_exp(self):
endpoint = "diffexp/obs" endpoint = "diffexp/obs"
url = f"{URL_BASE}{endpoint}" url = f"{URL_BASE}{endpoint}"
@@ -227,28 +148,6 @@ class EndPoints(unittest.TestCase):
result_data = result.json() result_data = result.json()
self.assertEqual(len(result_data), 10) self.assertEqual(len(result_data), 10)
def test_get_annotations_var(self):
endpoint = "annotations/var"
url = f"{URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["names"], ["name", "n_cells"])
self.assertEqual(len(result_data["data"]), 1838)
self.assertEqual(len(result_data["data"][0]), 3)
def test_get_annotations_var_keys(self):
endpoint = "annotations/var"
query = "annotation-name=n_cells"
url = f"{URL_BASE}{endpoint}?{query}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["names"], ["n_cells"])
self.assertEqual(len(result_data["data"][0]), 2)
def test_get_annotations_var_fbs(self): def test_get_annotations_var_fbs(self):
endpoint = "annotations/var" endpoint = "annotations/var"
url = f"{URL_BASE}{endpoint}" url = f"{URL_BASE}{endpoint}"
@@ -265,6 +164,23 @@ class EndPoints(unittest.TestCase):
self.assertEqual(len(df['columns']), df['n_cols']) self.assertEqual(len(df['columns']), df['n_cols'])
self.assertListEqual(df['col_idx'], ['name', 'n_cells']) self.assertListEqual(df['col_idx'], ['name', 'n_cells'])
def test_get_annotations_var_keys_fbs(self):
endpoint = "annotations/var"
query = "annotation-name=n_cells"
url = f"{URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 1838)
self.assertEqual(df['n_cols'], 1)
self.assertIsNotNone(df['columns'])
self.assertIsNotNone(df['col_idx'])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
self.assertListEqual(df['col_idx'], ['n_cells'])
def test_get_annotations_var_error(self): def test_get_annotations_var_error(self):
endpoint = "annotations/var" endpoint = "annotations/var"
query = "annotation-name=notakey" query = "annotation-name=notakey"
@@ -272,95 +188,36 @@ class EndPoints(unittest.TestCase):
result = self.session.get(url) result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST) self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
def test_put_annotations_var(self):
endpoint = "annotations/var"
url = f"{URL_BASE}{endpoint}"
var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}]}}}
result = self.session.put(url, json=var_filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["names"], ["name", "n_cells"])
self.assertEqual(len(result_data["data"]), 2)
def test_filter_put_annotations_var(self):
endpoint = "annotations/var"
query = "annotation-name=n_cells"
url = f"{URL_BASE}{endpoint}?{query}"
var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}]}}}
result = self.session.put(url, json=var_filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["names"], ["n_cells"])
self.assertEqual(len(result_data["data"][0]), 2)
self.assertEqual(len(result_data["data"]), 2)
def test_get_data(self):
for axis in ["obs", "var"]:
endpoint = f"data/{axis}"
query = "accept-type=application/json"
url = f"{URL_BASE}{endpoint}?{query}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data["obs"]), 2638)
def test_data_mimetype_error(self): def test_data_mimetype_error(self):
for axis in ["obs", "var"]: endpoint = f"data/var"
endpoint = f"data/{axis}" header = {"Accept": "xxx"}
query = "accept-type=xxx" url = f"{URL_BASE}{endpoint}"
url = f"{URL_BASE}{endpoint}?{query}" result = self.session.put(url, headers=header)
result = self.session.get(url) self.assertEqual(result.status_code, HTTPStatus.NOT_ACCEPTABLE)
self.assertEqual(result.status_code, HTTPStatus.NOT_ACCEPTABLE)
url = f"{URL_BASE}{endpoint}"
header = {"Accept": "sdkljfa;dsjalkj"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.NOT_ACCEPTABLE)
def test_json_default(self): def test_fbs_default(self):
for axis in ["obs", "var"]: endpoint = f"data/var"
endpoint = f"data/{axis}" url = f"{URL_BASE}{endpoint}"
url = f"{URL_BASE}{endpoint}" result = self.session.put(url)
result = self.session.get(url) self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
self.assertEqual(result.headers["Content-Type"], "application/json")
def test_data_filter(self):
for axis in ["obs", "var"]:
endpoint = f"data/{axis}"
query = "accept-type=application/json&obs:louvain=NK cells&obs:louvain=CD8 T cells&obs:n_counts=3000,*"
url = f"{URL_BASE}{endpoint}?{query}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data["obs"]), 38)
def test_data_json_put(self):
for axis in ["obs", "var"]:
endpoint = f"data/{axis}"
url = f"{URL_BASE}{endpoint}"
header = {"Accept": "application/json"}
obs_filter = {
"filter": {
"obs": {
"annotation_value": [
{"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
{"name": "n_counts", "min": 3000},
],
"index": [1, 99, [1000, 2000]],
}
}
}
result = self.session.put(url, headers=header, json=obs_filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data["obs"]), 15)
def test_data_put_fbs(self): def test_data_put_fbs(self):
endpoint = f"data/var"
url = f"{URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.session.put(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 1838)
self.assertIsNotNone(df['columns'])
self.assertListEqual(df['col_idx'].tolist(), [])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
def test_data_put_filter_fbs(self):
endpoint = f"data/var" endpoint = f"data/var"
url = f"{URL_BASE}{endpoint}" url = f"{URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"} header = {"Accept": "application/octet-stream"}
@@ -384,94 +241,16 @@ class EndPoints(unittest.TestCase):
self.assertListEqual(df['col_idx'].tolist(), [0, 1, 4]) self.assertListEqual(df['col_idx'].tolist(), [0, 1, 4])
def test_data_put_single_var(self): def test_data_put_single_var(self):
for axis in ["obs", "var"]: endpoint = f"data/var"
endpoint = f"data/{axis}"
url = f"{URL_BASE}{endpoint}"
header = {"Accept": "application/json"}
var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}}}
result = self.session.put(url, headers=header, json=var_filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
if axis == "obs":
self.assertEqual(len(result_data["obs"][0]), 2)
self.assertEqual(len(result_data["var"]), 1)
elif axis == "var":
self.assertEqual(len(result_data["obs"]), 2638)
self.assertEqual(len(result_data["var"][0]), 2639)
def test_cache(self):
endpoint = "annotations/var"
url = f"{URL_BASE}{endpoint}" url = f"{URL_BASE}{endpoint}"
f1 = { header = {"Accept": "application/octet-stream"}
"filter": { var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}}}
"var": { result = self.session.put(url, headers=header, json=var_filter)
"annotation_value": [
{
"name": "name",
"values": [
"HLA-DRB1",
"HLA-DQA1",
"HLA-DQB1",
"HLA-DPA1",
"HLA-DPB1",
"MS4A1",
"IL32",
"CCL5",
"CD79B",
"CD79A",
],
}
]
}
}
}
result = self.session.put(url, json=f1)
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json") self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
result_data1 = result.json() df = decode_fbs.decode_matrix_FBS(result.content)
f2 = { self.assertEqual(df["n_rows"], 2638)
"filter": { self.assertEqual(df["n_cols"], 1)
"var": {
"annotation_value": [
{
"name": "name",
"values": ["FGFBP2", "GZMA", "LTB", "PRF1", "CTSW", "GZMH", "CCL5", "CCL4", "CST7", "NKG7"],
}
]
}
}
}
result = self.session.put(url, json=f2)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data2 = result.json()
self.assertNotEqual(result_data1, result_data2)
def test_cache_nofilter(self):
endpoint = "annotations/var"
url = f"{URL_BASE}{endpoint}"
f1 = {"filter": {}}
result = self.session.put(url, json=f1)
self.assertEqual(result.status_code, HTTPStatus.OK)
result_data1 = result.json()
f2 = {
"filter": {
"var": {
"annotation_value": [
{
"name": "name",
"values": ["FGFBP2", "GZMA", "LTB", "PRF1", "CTSW", "GZMH", "CCL5", "CCL4", "CST7", "NKG7"],
}
]
}
}
}
result = self.session.put(url, json=f2)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data2 = result.json()
self.assertNotEqual(result_data1, result_data2)
def test_static(self): def test_static(self):
endpoint = "static" endpoint = "static"
server/test/test_filter.py
-82
View File
@@ -1,82 +0,0 @@
import json
from os import path
import unittest
from numpy import float32, int32
from werkzeug.datastructures import ImmutableMultiDict
from server.app.util.filter import _convert_variable, parse_filter, QueryStringError
class UtilTest(unittest.TestCase):
"""Test Case for endpoints"""
def setUp(self):
with open(path.join(path.dirname(__file__), "schema.json")) as fh:
schema = json.load(fh)
self.schema = schema["annotations"]
def test_convert(self):
five = _convert_variable("int32", "5")
self.assertEqual(five, int32(5))
def test_convert_zero(self):
zero = _convert_variable("int32", "0")
self.assertEqual(zero, 0)
def test_convert_float(self):
str_to_convert = "4.38719237129"
val = _convert_variable("float32", str_to_convert)
self.assertAlmostEqual(val, float32(str_to_convert))
def test_convert_bool(self):
str_to_convert = "false"
val = _convert_variable("boolean", str_to_convert)
self.assertFalse(val)
str_to_convert = "true"
val = _convert_variable("boolean", str_to_convert)
self.assertTrue(val)
str_to_convert = "0"
with self.assertRaises(AssertionError):
val = _convert_variable("boolean", str_to_convert)
def test_empty_convert(self):
empty = _convert_variable("int32", None)
self.assertIsNone(empty)
def test_bad_convert(self):
with self.assertRaises(ValueError):
_convert_variable("int32", "5.5")
def test_bad_datatype(self):
with self.assertRaises(AssertionError):
_convert_variable("jkasdslkja", 1)
def test_complex_filter(self):
filter_dict = ImmutableMultiDict(
[("obs:louvain", "NK cells"), ("obs:louvain", "CD8 T cells"), ("obs:n_counts", "3000,*")]
)
filter_ = parse_filter(filter_dict, self.schema)
self.assertIn("obs", filter_)
self.assertEqual(
filter_["obs"]["annotation_value"],
[
{"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
{"name": "n_counts", "max": None, "min": 3000.0},
],
)
def test_bad_filter(self):
bad_annotation_type = ImmutableMultiDict([("obs:tissue", "lung")])
with self.assertRaises(QueryStringError):
parse_filter(bad_annotation_type, self.schema)
bad_axis = ImmutableMultiDict([("xyz:n_genes", "100,1000")])
with self.assertRaises(QueryStringError):
parse_filter(bad_axis, self.schema)
def test_boolean_filter(self):
schema = {"obs": [{"name": "bool_filter", "type": "boolean"}]}
filter_dict = ImmutableMultiDict([("obs:bool_filter", "false")])
filter_ = parse_filter(filter_dict, schema)
self.assertIn("obs", filter_)
self.assertEqual(filter_["obs"]["annotation_value"], [{"name": "bool_filter", "values": [False]}])
server/test/test_nan_rest.py
+29 -6
View File
@@ -2,6 +2,9 @@ from http import HTTPStatus
from subprocess import Popen from subprocess import Popen
import unittest import unittest
import time import time
import math
import decode_fbs
import requests import requests
@@ -43,9 +46,29 @@ class WithNaNs(unittest.TestCase):
result = self.session.get(url) result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
def test_errors(self): def test_data(self):
endpoints = ["annotations/obs", "annotations/var", "data/obs", "data/var"] endpoint = "data/var"
for endpoint in endpoints: url = f"{URL_BASE}{endpoint}"
url = f"{URL_BASE}{endpoint}" result = self.session.put(url)
result = self.session.get(url) self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.status_code, HTTPStatus.INTERNAL_SERVER_ERROR) self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertTrue(math.isnan(df["columns"][3][3]))
def test_annotation_obs(self):
endpoint = "annotations/obs"
url = f"{URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertTrue(math.isnan(df["columns"][2][0]))
def test_annotation_var(self):
endpoint = "annotations/var"
url = f"{URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertTrue(math.isnan(df["columns"][2][0]))
server/test/test_nan_scanpy_engine.py
+10 -11
View File
@@ -1,4 +1,3 @@
import json
import pytest import pytest
import unittest import unittest
import warnings import warnings
@@ -7,7 +6,7 @@ import math
import decode_fbs import decode_fbs
from server.app.scanpy_engine.scanpy_engine import ScanpyEngine from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
from server.app.util.errors import JSONEncodingValueError from server.app.util.errors import FilterError
class NaNTest(unittest.TestCase): class NaNTest(unittest.TestCase):
@@ -42,10 +41,15 @@ class NaNTest(unittest.TestCase):
self.assertEqual(data_frame_var["n_cols"], 100) self.assertEqual(data_frame_var["n_cols"], 100)
self.assertTrue(math.isnan(data_frame_var["columns"][3][3])) self.assertTrue(math.isnan(data_frame_var["columns"][3][3]))
with pytest.raises(JSONEncodingValueError): with pytest.raises(FilterError):
json.loads(self.data.data_frame(None, "obs")) self.data.data_frame_to_fbs_matrix("an erroneous filter", "var")
with pytest.raises(JSONEncodingValueError): with pytest.raises(FilterError):
json.loads(self.data.data_frame(None, "var")) filter_ = {
"filter": {
"obs": {"index": [1, 99, [200, 300]]}
}
}
self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
def test_dataframe_obs_not_implemented(self): def test_dataframe_obs_not_implemented(self):
with self.assertRaises(ValueError) as cm: with self.assertRaises(ValueError) as cm:
@@ -65,8 +69,3 @@ class NaNTest(unittest.TestCase):
self.assertEqual(annotations["col_idx"], ["name", "n_cells", "var_with_nans"]) self.assertEqual(annotations["col_idx"], ["name", "n_cells", "var_with_nans"])
self.assertEqual(annotations["n_rows"], 100) self.assertEqual(annotations["n_rows"], 100)
self.assertTrue(math.isnan(annotations["columns"][2][0])) self.assertTrue(math.isnan(annotations["columns"][2][0]))
with pytest.raises(JSONEncodingValueError):
json.loads(self.data.annotation(None, "obs"))
with pytest.raises(JSONEncodingValueError):
json.loads(self.data.annotation(None, "var"))
server/test/test_scanpy_engine.py
+81 -114
View File
@@ -3,11 +3,13 @@ from os import path
import pytest import pytest
import time import time
import unittest import unittest
import decode_fbs
import numpy as np import numpy as np
from pandas import Series from pandas import Series
from server.app.scanpy_engine.scanpy_engine import ScanpyEngine from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
from server.app.util.errors import FilterError
class UtilTest(unittest.TestCase): class UtilTest(unittest.TestCase):
@@ -45,55 +47,29 @@ class UtilTest(unittest.TestCase):
def test_filter_idx(self): def test_filter_idx(self):
filter_ = { filter_ = {
"filter": { "filter": {
"var": {"index": [1, 99, [200, 300]]}, "var": {"index": [1, 99, [200, 300]]}
"obs": {"index": [1, 99, [1000, 2000]]},
} }
} }
data = self.data.filter_dataframe(filter_["filter"]) fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
self.assertEqual(data.shape, (1002, 102)) data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(data["n_rows"], 2638)
def test_filter_annotation(self): self.assertEqual(data["n_cols"], 102)
filter_ = {
"filter": {
"obs": {
"annotation_value": [
{"name": "louvain", "values": ["NK cells", "CD8 T cells"]}
]
}
}
}
data = self.data.filter_dataframe(filter_["filter"])
self.assertEqual(data.shape, (470, 1838))
filter_ = {
"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}
}
data = self.data.filter_dataframe(filter_["filter"])
self.assertEqual(data.shape, (497, 1838))
def test_filter_annotation_no_uns(self):
filter_ = {
"filter": {
"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}
}
}
data = self.data.filter_dataframe(filter_["filter"])
self.assertEqual(data.shape[1], 1)
def test_filter_complex(self): def test_filter_complex(self):
filter_ = { filter_ = {
"filter": { "filter": {
"var": {"index": [1, 99, [200, 300]]}, "var": {
"obs": {
"annotation_value": [ "annotation_value": [
{"name": "louvain", "values": ["NK cells", "CD8 T cells"]}, {"name": "n_cells", "min": 10}
{"name": "n_counts", "min": 3000},
], ],
"index": [1, 99, [1000, 2000]], "index": [1, 99, [200, 300]]
}, }
} }
} }
data = self.data.filter_dataframe(filter_["filter"]) fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
self.assertEqual(data.shape, (15, 102)) data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(data["n_rows"], 2638)
self.assertEqual(data["n_cols"], 91)
def test_obs_and_var_names(self): def test_obs_and_var_names(self):
self.assertEqual(np.sum(self.data.data.var["name"].isna()), 0) self.assertEqual(np.sum(self.data.data.var["name"].isna()), 0)
@@ -119,60 +95,42 @@ class UtilTest(unittest.TestCase):
) )
def test_layout(self): def test_layout(self):
layout = json.loads(self.data.layout(None)) fbs = self.data.layout_to_fbs_matrix()
self.assertEqual(layout["layout"]["ndims"], 2) layout = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(len(layout["layout"]["coordinates"]), 2638) self.assertEqual(layout["n_cols"], 2)
self.assertEqual(layout["layout"]["coordinates"][0][0], 0) self.assertEqual(layout["n_rows"], 2638)
for idx, val in enumerate(layout["layout"]["coordinates"]):
self.assertLessEqual(val[1], 1) X = layout["columns"][0]
self.assertLessEqual(val[2], 1) self.assertTrue((X >= 0).all() and (X <= 1).all())
Y = layout["columns"][1]
self.assertTrue((Y >= 0).all() and (Y <= 1).all())
def test_annotations(self): def test_annotations(self):
annotations = json.loads(self.data.annotation(None, "obs")) fbs = self.data.annotation_to_fbs_matrix("obs")
annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations["n_rows"], 2638)
self.assertEqual(annotations["n_cols"], 5)
self.assertEqual( self.assertEqual(
annotations["names"], annotations["col_idx"],
["name", "n_genes", "percent_mito", "n_counts", "louvain"], ["name", "n_genes", "percent_mito", "n_counts", "louvain"],
) )
self.assertEqual(len(annotations["data"]), 2638)
annotations = json.loads(self.data.annotation(None, "var")) fbs = self.data.annotation_to_fbs_matrix("var")
self.assertEqual(annotations["names"], ["name", "n_cells"]) annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(len(annotations["data"]), 1838) self.assertEqual(annotations['n_rows'], 1838)
self.assertEqual(annotations['n_cols'], 2)
self.assertEqual(annotations["col_idx"], ["name", "n_cells"])
def test_annotation_fields(self): def test_annotation_fields(self):
annotations = json.loads( fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"])
self.data.annotation(None, "obs", ["n_genes", "n_counts"]) annotations = decode_fbs.decode_matrix_FBS(fbs)
) self.assertEqual(annotations["n_rows"], 2638)
self.assertEqual(annotations["names"], ["n_genes", "n_counts"]) self.assertEqual(annotations['n_cols'], 2)
self.assertEqual(len(annotations["data"]), 2638)
annotations = json.loads(self.data.annotation(None, "var", ["name"]))
self.assertEqual(annotations["names"], ["name"])
self.assertEqual(len(annotations["data"]), 1838)
def test_filtered_annotation(self): fbs = self.data.annotation_to_fbs_matrix("var", ["name"])
filter_ = { annotations = decode_fbs.decode_matrix_FBS(fbs)
"filter": { self.assertEqual(annotations['n_rows'], 1838)
"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}, self.assertEqual(annotations['n_cols'], 1)
"var": {
"annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}]
},
}
}
annotations = json.loads(self.data.annotation(filter_["filter"], "obs"))
self.assertEqual(
annotations["names"],
["name", "n_genes", "percent_mito", "n_counts", "louvain"],
)
self.assertEqual(len(annotations["data"]), 497)
annotations = json.loads(self.data.annotation(filter_["filter"], "var"))
self.assertEqual(annotations["names"], ["name", "n_cells"])
self.assertEqual(len(annotations["data"]), 2)
def test_filtered_layout(self):
filter_ = {
"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}
}
layout = json.loads(self.data.layout(filter_["filter"]))
self.assertEqual(len(layout["layout"]["coordinates"]), 497)
def test_diffexp_topN(self): def test_diffexp_topN(self):
f1 = {"filter": {"obs": {"index": [[0, 500]]}}} f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
@@ -183,42 +141,51 @@ class UtilTest(unittest.TestCase):
self.assertEqual(len(result), 20) self.assertEqual(len(result), 20)
def test_data_frame(self): def test_data_frame(self):
data_frame_obs = json.loads(self.data.data_frame(None, "obs")) fbs = self.data.data_frame_to_fbs_matrix(None, "var")
self.assertEqual(len(data_frame_obs["var"]), 1838) data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(len(data_frame_obs["obs"]), 2638) self.assertEqual(data["n_rows"], 2638)
data_frame_var = json.loads(self.data.data_frame(None, "var")) self.assertEqual(data["n_cols"], 1838)
self.assertEqual(len(data_frame_var["var"]), 1838)
self.assertEqual(len(data_frame_var["obs"]), 2638) with self.assertRaises(ValueError):
self.data.data_frame_to_fbs_matrix(None, "obs")
def test_filtered_data_frame(self): def test_filtered_data_frame(self):
filter_ = {
"filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 100}]}}
}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(data["n_rows"], 2638)
self.assertEqual(data["n_cols"], 1040)
filter_ = { filter_ = {
"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}} "filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}
} }
data_frame_obs = json.loads(self.data.data_frame(filter_["filter"], "obs")) with self.assertRaises(FilterError):
self.assertEqual(len(data_frame_obs["var"]), 1838) self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
self.assertEqual(len(data_frame_obs["obs"]), 497)
self.assertIsInstance(data_frame_obs["obs"][0], (list, tuple))
self.assertEqual(type(data_frame_obs["var"][0]), int)
data_frame_var = json.loads(self.data.data_frame(filter_["filter"], "var"))
self.assertEqual(len(data_frame_var["var"]), 1838)
self.assertEqual(len(data_frame_var["obs"]), 497)
self.assertIsInstance(data_frame_var["var"][0], (list, tuple))
self.assertEqual(type(data_frame_var["obs"][0]), int)
def test_data_single_gene(self): def test_data_named_gene(self):
for axis in ["obs", "var"]: filter_ = {
filter_ = { "filter": {
"filter": { "var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}
"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}
}
} }
data_frame_var = json.loads(self.data.data_frame(filter_["filter"], axis)) }
if axis == "obs": fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
self.assertEqual(type(data_frame_var["var"][0]), int) data = decode_fbs.decode_matrix_FBS(fbs)
self.assertIsInstance(data_frame_var["obs"][0], (list, tuple)) self.assertEqual(data["n_rows"], 2638)
elif axis == "var": self.assertEqual(data["n_cols"], 1)
self.assertEqual(type(data_frame_var["obs"][0]), int) self.assertEqual(data["col_idx"], [4])
self.assertIsInstance(data_frame_var["var"][0], (list, tuple))
filter_ = {
"filter": {
"var": {"annotation_value": [{"name": "name", "values": ["SPEN", "TYMP", "PRMT2"]}]}
}
}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(data["n_rows"], 2638)
self.assertEqual(data["n_cols"], 3)
self.assertTrue((data["col_idx"] == [15, 1818, 1837]).all())
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()