fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-10-07 03:48:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Black -- formatter for python (#508)

Browse Source 
* Add black

* use black to format code

* Black version
This commit is contained in:
Charlotte Weaver
2018-12-12 09:44:47 -08:00
committed by GitHub
parent a847951658
commit 83154577e4
18 changed files with 636 additions and 791 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/app/app.py
+10 -6
View File
@@ -14,16 +14,14 @@ REACTIVE_LIMIT = 1_000_000
app = Flask(__name__, static_folder="web/static")
app.json_encoder = Float32JSONEncoder
cache = Cache(app, config={"CACHE_TYPE": "simple", "CACHE_DEFAULT_TIMEOUT": 860000})
cache = Cache(app, config={"CACHE_TYPE": "simple", "CACHE_DEFAULT_TIMEOUT": 860_000})
Compress(app)
CORS(app)
# Config
SECRET_KEY = os.environ.get("CXG_SECRET_KEY", default="SparkleAndShine")
app.config.update(
SECRET_KEY=SECRET_KEY,
)
app.config.update(SECRET_KEY=SECRET_KEY)
# Application Data
data = None
@@ -36,7 +34,13 @@ docs.append(resources.get_swagger_doc())
app.register_blueprint(webapp.bp)
app.register_blueprint(resources.blueprint)
app.register_blueprint(
get_swagger_blueprint(docs, "/api/swagger", produces=["application/json"], title="cellxgene rest api",
description="An API connecting ExpressionMatrix2 clustering algorithm to cellxgene"))
get_swagger_blueprint(
docs,
"/api/swagger",
produces=["application/json"],
title="cellxgene rest api",
description="An API connecting ExpressionMatrix2 clustering algorithm to cellxgene",
)
)
app.add_url_rule("/", endpoint="index")
server/app/driver/driver.py
+2 -6
View File
@@ -11,7 +11,6 @@ Sort order for methods
class CXGDriver(metaclass=ABCMeta):
def __init__(self, data, args):
self.data = self._load_data(data)
self.layout_method = args["layout"]
@@ -24,11 +23,8 @@ class CXGDriver(metaclass=ABCMeta):
def features(self):
features = {
"cluster": {"available": False},
"layout": {
"obs": {"available": False},
"var": {"available": False},
},
"diffexp": {"available": False}
"layout": {"obs": {"available": False}, "var": {"available": False}},
"diffexp": {"available": False},
}
# TODO - Interactive limit should be generated from the actual available methods see GH issue #94
if self.layout_method:
server/app/rest_api/rest.py
+327 -426
View File
@@ -2,9 +2,7 @@ from http import HTTPStatus
import pkg_resources
import warnings
from flask import (
Blueprint, current_app, jsonify, make_response, request
)
from flask import Blueprint, current_app, jsonify, make_response, request
from flask_restful_swagger_2 import Api, swagger, Resource
from werkzeug.datastructures import ImmutableMultiDict
@@ -23,83 +21,78 @@ Sort order for routes
class SchemaAPI(Resource):
@swagger.doc({
"summary": "get schema for dataframe and annotations",
"tags": ["initialize"],
"parameters": [],
"responses": {
"200": {
"description": "schema",
"examples": {
"application/json": {
"schema": {
"dataframe": {
"nObs": 383,
"nVar": 19944,
"type": "float32"
},
"annotations": {
"obs": [
{"name": "name", "type": "string"},
{"name": "tissue_type", "type": "string"},
{"name": "num_reads", "type": "int32"},
{"name": "sample_name", "type": "string"},
{
"name": "clusters",
"type": "categorical",
"categories": [99, 1, "unknown cluster"]
},
{"name": "QScore", "type": "float32"}
],
"var": [
{"name": "name", "type": "string"},
{"name": "gene", "type": "string"}
]
@swagger.doc(
{
"summary": "get schema for dataframe and annotations",
"tags": ["initialize"],
"parameters": [],
"responses": {
"200": {
"description": "schema",
"examples": {
"application/json": {
"schema": {
"dataframe": {"nObs": 383, "nVar": 19944, "type": "float32"},
"annotations": {
"obs": [
{"name": "name", "type": "string"},
{"name": "tissue_type", "type": "string"},
{"name": "num_reads", "type": "int32"},
{"name": "sample_name", "type": "string"},
{
"name": "clusters",
"type": "categorical",
"categories": [99, 1, "unknown cluster"],
},
{"name": "QScore", "type": "float32"},
],
"var": [{"name": "name", "type": "string"}, {"name": "gene", "type": "string"}],
},
}
}
}
},
}
}
},
}
})
)
def get(self):
return make_response(jsonify({"schema": current_app.data.schema}), HTTPStatus.OK)
class ConfigAPI(Resource):
@swagger.doc({
"summary": "Configuration information to assist in front-end adaptation"
" to underlying engine, available functionality, interactive time limits, etc",
"tags": ["initialize"],
"parameters": [],
"responses": {
"200": {
"description": "schema",
"examples": {
"application/json": {
"config": {
"features": [
{"method": "POST", "path": "/cluster/", "available": False},
{
"method": "POST",
"path": "/layout/obs",
"available": True,
"interactiveLimit": 10000
@swagger.doc(
{
"summary": "Configuration information to assist in front-end adaptation"
" to underlying engine, available functionality, interactive time limits, etc",
"tags": ["initialize"],
"parameters": [],
"responses": {
"200": {
"description": "schema",
"examples": {
"application/json": {
"config": {
"features": [
{"method": "POST", "path": "/cluster/", "available": False},
{
"method": "POST",
"path": "/layout/obs",
"available": True,
"interactiveLimit": 10000,
},
{"method": "POST", "path": "/layout/var", "available": False},
],
"displayNames": {
"engine": "ScanPy version 1.33",
"dataset": "/home/joe/mouse/blorth.csv",
},
{"method": "POST", "path": "/layout/var", "available": False}
],
"displayNames": {
"engine": "ScanPy version 1.33",
"dataset": "/home/joe/mouse/blorth.csv"
},
}
}
}
},
}
}
},
}
})
)
def get(self):
config = {
"config": {
@@ -111,50 +104,48 @@ class ConfigAPI(Resource):
],
"displayNames": {
"engine": f"cellxgene Scanpy engine version {pkg_resources.get_distribution('cellxgene').version}",
"dataset": current_app.config["DATASET_TITLE"]
"dataset": current_app.config["DATASET_TITLE"],
},
"parameters": {
"max_category_items": current_app.data.max_category_items
}
"parameters": {"max_category_items": current_app.data.max_category_items},
}
}
return make_response(jsonify(config), HTTPStatus.OK)
class AnnotationsObsAPI(Resource):
@swagger.doc({
"summary": "Fetch annotations (metadata) for all observations.",
"tags": ["annotations"],
"parameters": [{
"in": "query",
"name": "annotation-name",
"type": "string",
"description": "list of 1 or more annotation names"
}],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": [
"tissue_type", "sex", "num_reads", "clusters"
],
"data": [
[0, "lung", "F", 39844, 99],
[1, "heart", "M", 83, 1],
[49, "spleen", None, 2, "unknown cluster"],
]
}
@swagger.doc(
{
"summary": "Fetch annotations (metadata) for all observations.",
"tags": ["annotations"],
"parameters": [
{
"in": "query",
"name": "annotation-name",
"type": "string",
"description": "list of 1 or more annotation names",
}
],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": ["tissue_type", "sex", "num_reads", "clusters"],
"data": [
[0, "lung", "F", 39844, 99],
[1, "heart", "M", 83, 1],
[49, "spleen", None, 2, "unknown cluster"],
],
}
},
},
"400": {
"description": "one or more of the annotation-name identifiers were not associated with an "
"annotation name"
},
},
"400": {
"description": "one or more of the annotation-name identifiers were not associated with an "
"annotation name"
}
}
})
)
def get(self):
fields = request.args.getlist("annotation-name", None)
try:
@@ -168,47 +159,40 @@ class AnnotationsObsAPI(Resource):
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
@swagger.doc({
"summary": "Fetch annotations (metadata) for filtered subset of observations.",
"tags": ["annotations"],
"parameters": [
{
"in": "query",
"name": "annotation-name",
"type": "string",
"description": "list of 1 or more annotation names"
@swagger.doc(
{
"summary": "Fetch annotations (metadata) for filtered subset of observations.",
"tags": ["annotations"],
"parameters": [
{
"in": "query",
"name": "annotation-name",
"type": "string",
"description": "list of 1 or more annotation names",
},
{"name": "filter", "description": "Complex Filter", "in": "body", "schema": FilterModel},
],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": ["tissue_type", "sex", "num_reads", "clusters"],
"data": [
[0, "lung", "F", 39844, 99],
[1, "heart", "M", 83, 1],
[49, "spleen", None, 2, "unknown cluster"],
],
}
},
},
"400": {
"description": "malformed filter or one or more of the annotation-name identifiers were"
"not associated with an annotation name"
},
},
{
"name": "filter",
"description": "Complex Filter",
"in": "body",
"schema": FilterModel
}
],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": [
"tissue_type", "sex", "num_reads", "clusters"
],
"data": [
[0, "lung", "F", 39844, 99],
[1, "heart", "M", 83, 1],
[49, "spleen", None, 2, "unknown cluster"],
]
}
}
},
"400": {
"description": "malformed filter or one or more of the annotation-name identifiers were"
"not associated with an annotation name"
}
}
})
)
def put(self):
fields = request.args.getlist("annotation-name", None)
try:
@@ -226,38 +210,35 @@ class AnnotationsObsAPI(Resource):
class AnnotationsVarAPI(Resource):
@swagger.doc({
"summary": "Fetch annotations (metadata) for all variables.",
"tags": ["annotations"],
"parameters": [{
"in": "query",
"name": "annotation-name",
"type": "string",
"description": "list of 1 or more annotation names"
}],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": [
"name", "category"
],
"data": [
[0, "ATAD3C", 1],
[1, "RER1", None],
[49, "S100B", 6]
]
}
@swagger.doc(
{
"summary": "Fetch annotations (metadata) for all variables.",
"tags": ["annotations"],
"parameters": [
{
"in": "query",
"name": "annotation-name",
"type": "string",
"description": "list of 1 or more annotation names",
}
],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": ["name", "category"],
"data": [[0, "ATAD3C", 1], [1, "RER1", None], [49, "S100B", 6]],
}
},
},
"400": {
"description": "one or more of the annotation-name identifiers were not associated with an"
" annotation name"
},
},
"400": {
"description": "one or more of the annotation-name identifiers were not associated with an"
" annotation name"
}
}
})
)
def get(self):
fields = request.args.getlist("annotation-name", None)
try:
@@ -271,45 +252,36 @@ class AnnotationsVarAPI(Resource):
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
@swagger.doc({
"summary": "Fetch annotations (metadata) for filtered subset of variables.",
"tags": ["annotations"],
"parameters": [
{
"in": "query",
"name": "annotation-name",
"type": "string",
"description": "list of 1 or more annotation names"
@swagger.doc(
{
"summary": "Fetch annotations (metadata) for filtered subset of variables.",
"tags": ["annotations"],
"parameters": [
{
"in": "query",
"name": "annotation-name",
"type": "string",
"description": "list of 1 or more annotation names",
},
{"name": "filter", "description": "Complex Filter", "in": "body", "schema": FilterModel},
],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": ["name", "category"],
"data": [[0, "ATAD3C", 1], [1, "RER1", None], [49, "S100B", 6]],
}
},
},
"400": {
"description": "malformed filter or one or more of the annotation-name identifiers were"
"not associated with an annotation name"
},
},
{
"name": "filter",
"description": "Complex Filter",
"in": "body",
"schema": FilterModel
}
],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": [
"name", "category"
],
"data": [
[0, "ATAD3C", 1],
[1, "RER1", None],
[49, "S100B", 6]
]
}
}
},
"400": {
"description": "malformed filter or one or more of the annotation-name identifiers were"
"not associated with an annotation name"
}
}
})
)
def put(self):
fields = request.args.getlist("annotation-name", None)
try:
@@ -327,57 +299,39 @@ class AnnotationsVarAPI(Resource):
class DataObsAPI(Resource):
@swagger.doc({
"summary": "Get data (expression values) from the dataframe.",
"tags": ["data"],
"parameters": [
{
"in": "query",
"name": "filter",
"type": "string",
"description": "axis:key:value"
},
{
"in": "query",
"name": "accept-type",
"type": "string",
"description": "MIME type"
},
],
"responses": {
"200": {
"description": "expression",
"examples": {
"application/json": {
"var": [0, 20000],
"obs": [
[1, 39483, 3902, 203, 0, 0, 28]
]
}
}
},
"400": {
"description": "Malformed filter"
},
"406": {
"description": "Unacceptable MIME type"
@swagger.doc(
{
"summary": "Get data (expression values) from the dataframe.",
"tags": ["data"],
"parameters": [
{"in": "query", "name": "filter", "type": "string", "description": "axis:key:value"},
{"in": "query", "name": "accept-type", "type": "string", "description": "MIME type"},
],
"responses": {
"200": {
"description": "expression",
"examples": {"application/json": {"var": [0, 20000], "obs": [[1, 39483, 3902, 203, 0, 0, 28]]}},
},
"400": {"description": "Malformed filter"},
"406": {"description": "Unacceptable MIME type"},
},
}
})
)
def get(self):
accept_type = request.args.get("accept-type", None)
# request.args is immutable
args = request.args.copy()
args.pop("accept-type", None)
try:
filter_ = parse_filter(ImmutableMultiDict(args), current_app.data.schema['annotations'])
filter_ = parse_filter(ImmutableMultiDict(args), current_app.data.schema["annotations"])
except QueryStringError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
# TODO support CSV
try:
# TODO store mime_type when more than one is supported
get_mime_type(acceptable_types=["application/json"], query_param=accept_type,
header=request.accept_mimetypes)
get_mime_type(
acceptable_types=["application/json"], query_param=accept_type, header=request.accept_mimetypes
)
except MimeTypeError as e:
return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
try:
@@ -389,37 +343,21 @@ class DataObsAPI(Resource):
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
@swagger.doc({
"summary": "Get data (expression values) from the dataframe.",
"tags": ["data"],
"parameters": [
{
'name': 'filter',
'description': 'Complex Filter',
'in': 'body',
'schema': FilterModel
}
],
"responses": {
"200": {
"description": "expression",
"examples": {
"application/json": {
"var": [0, 20000],
"obs": [
[1, 39483, 3902, 203, 0, 0, 28]
]
}
}
},
"400": {
"description": "Malformed filter"
},
"406": {
"description": "Unacceptable MIME type"
@swagger.doc(
{
"summary": "Get data (expression values) from the dataframe.",
"tags": ["data"],
"parameters": [{"name": "filter", "description": "Complex Filter", "in": "body", "schema": FilterModel}],
"responses": {
"200": {
"description": "expression",
"examples": {"application/json": {"var": [0, 20000], "obs": [[1, 39483, 3902, 203, 0, 0, 28]]}},
},
"400": {"description": "Malformed filter"},
"406": {"description": "Unacceptable MIME type"},
},
}
})
)
def put(self):
if not request.accept_mimetypes.best_match(["application/json", "text/csv"]):
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
@@ -428,8 +366,9 @@ class DataObsAPI(Resource):
except MimeTypeError as e:
return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
try:
return make_response((jsonify(current_app.data.data_frame(request.get_json()["filter"], axis=Axis.OBS))),
HTTPStatus.OK)
return make_response(
(jsonify(current_app.data.data_frame(request.get_json()["filter"], axis=Axis.OBS))), HTTPStatus.OK
)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except ValueError as e:
@@ -439,55 +378,37 @@ class DataObsAPI(Resource):
class DataVarAPI(Resource):
@swagger.doc({
"summary": "Get data (expression values) from the dataframe.",
"tags": ["data"],
"parameters": [
{
"in": "query",
"name": "filter",
"type": "string",
"description": "axis:key:value"
},
{
"in": "query",
"name": "accept-type",
"type": "string",
"description": "MIME type"
},
],
"responses": {
"200": {
"description": "expression",
"examples": {
"application/json": {
"obs": [0, 20000],
"var": [
[1, 39483, 3902, 203, 0, 0, 28]
]
}
}
},
"400": {
"description": "Malformed filter"
},
"406": {
"description": "Unacceptable MIME type"
@swagger.doc(
{
"summary": "Get data (expression values) from the dataframe.",
"tags": ["data"],
"parameters": [
{"in": "query", "name": "filter", "type": "string", "description": "axis:key:value"},
{"in": "query", "name": "accept-type", "type": "string", "description": "MIME type"},
],
"responses": {
"200": {
"description": "expression",
"examples": {"application/json": {"obs": [0, 20000], "var": [[1, 39483, 3902, 203, 0, 0, 28]]}},
},
"400": {"description": "Malformed filter"},
"406": {"description": "Unacceptable MIME type"},
},
}
})
)
def get(self):
accept_type = request.args.get("accept-type", None)
# request.args is immutable
args = request.args.copy()
args.pop("accept-type", None)
try:
filter_ = parse_filter(ImmutableMultiDict(args), current_app.data.schema['annotations'])
filter_ = parse_filter(ImmutableMultiDict(args), current_app.data.schema["annotations"])
except QueryStringError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
try:
get_mime_type(acceptable_types=["application/json"], query_param=accept_type,
header=request.accept_mimetypes)
get_mime_type(
acceptable_types=["application/json"], query_param=accept_type, header=request.accept_mimetypes
)
except MimeTypeError as e:
return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
try:
@@ -499,37 +420,21 @@ class DataVarAPI(Resource):
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
@swagger.doc({
"summary": "Get data (expression values) from the dataframe.",
"tags": ["data"],
"parameters": [
{
'name': 'filter',
'description': 'Complex Filter',
'in': 'body',
'schema': FilterModel
}
],
"responses": {
"200": {
"description": "expression",
"examples": {
"application/json": {
"obs": [0, 20000],
"var": [
[1, 39483, 3902, 203, 0, 0, 28]
]
}
}
},
"400": {
"description": "Malformed filter"
},
"406": {
"description": "Unacceptable MIME type"
@swagger.doc(
{
"summary": "Get data (expression values) from the dataframe.",
"tags": ["data"],
"parameters": [{"name": "filter", "description": "Complex Filter", "in": "body", "schema": FilterModel}],
"responses": {
"200": {
"description": "expression",
"examples": {"application/json": {"obs": [0, 20000], "var": [[1, 39483, 3902, 203, 0, 0, 28]]}},
},
"400": {"description": "Malformed filter"},
"406": {"description": "Unacceptable MIME type"},
},
}
})
)
def put(self):
if not request.accept_mimetypes.best_match(["application/json", "text/csv"]):
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
@@ -539,8 +444,9 @@ class DataVarAPI(Resource):
except MimeTypeError as e:
return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
try:
return make_response((jsonify(current_app.data.data_frame(request.get_json()["filter"], axis=Axis.VAR))),
HTTPStatus.OK)
return make_response(
(jsonify(current_app.data.data_frame(request.get_json()["filter"], axis=Axis.VAR))), HTTPStatus.OK
)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except ValueError as e:
@@ -550,67 +456,64 @@ class DataVarAPI(Resource):
class DiffExpObsAPI(Resource):
@swagger.doc({
"summary": "Generate differential expression (DE) statistics for two specified subsets of data, "
"as indicated by the two provided observation complex filters",
"tags": ["diffexp"],
# TODO sort out params
# "parameters": [
# # {
# # "in": "body",
# # "name": "mode",
# # "type": "string",
# # "required": True,
# # "description": "topN or varFilter"
# # },
# {
# "in": "query",
# "name": "count",
# "type": "int32",
# "description": "TopN mode: how many vars to return"
# },
# {
# "in": "body",
# "name": "varFilter",
# "schema": FilterModel,
# "description": "varFilter: Complex filter, only var for which vars to return"
# },
# {
# "in": "body",
# "name": "set1",
# "schema": FilterModel,
# "required": True,
# "description": "Complex filter, only obs - observations in set1"
# },
# {
# "in": "body",
# "name": "set2",
# "schema": FilterModel,
# "description": "Complex filter, only obs - observations in set2. If not included, inverse of set1."
# },
# ],
"responses": {
"200": {
"description": "Statistics are encoded as an array of arrays, with fields ordered as: "
"varIndex, logfoldchange, pVal, pValAdj",
"examples": {
"application/json": [
[328, -2.569489, 2.655706e-63, 3.642036e-57],
[1250, -2.569489, 2.655706e-63, 3.642036e-57],
]
}
@swagger.doc(
{
"summary": "Generate differential expression (DE) statistics for two specified subsets of data, "
"as indicated by the two provided observation complex filters",
"tags": ["diffexp"],
# TODO sort out params
# "parameters": [
# # {
# # "in": "body",
# # "name": "mode",
# # "type": "string",
# # "required": True,
# # "description": "topN or varFilter"
# # },
# {
# "in": "query",
# "name": "count",
# "type": "int32",
# "description": "TopN mode: how many vars to return"
# },
# {
# "in": "body",
# "name": "varFilter",
# "schema": FilterModel,
# "description": "varFilter: Complex filter, only var for which vars to return"
# },
# {
# "in": "body",
# "name": "set1",
# "schema": FilterModel,
# "required": True,
# "description": "Complex filter, only obs - observations in set1"
# },
# {
# "in": "body",
# "name": "set2",
# "schema": FilterModel,
# "description": "Complex filter, only obs - observations in set2.
# If not included, inverse of set1."
# },
# ],
"responses": {
"200": {
"description": "Statistics are encoded as an array of arrays, with fields ordered as: "
"varIndex, logfoldchange, pVal, pValAdj",
"examples": {
"application/json": [
[328, -2.569_489, 2.655_706e-63, 3.642_036e-57],
[1250, -2.569_489, 2.655_706e-63, 3.642_036e-57],
]
},
},
"400": {"description": "malformed filter"},
"403": {"description": "non-interactive request"},
"501": {"description": "diffexp is not implemented"},
},
"400": {
"description": "malformed filter"
},
"403": {
"description": "non-interactive request"
},
"501": {
"description": "diffexp is not implemented"
}
}
})
)
def post(self):
args = request.get_json()
# confirm mode is present and legal
@@ -645,8 +548,9 @@ class DiffExpObsAPI(Resource):
# mode=topN
count = args.get("count", None)
try:
diffexp = current_app.data.diffexp_topN(set1_filter, set2_filter, count,
current_app.data.features["diffexp"]["interactiveLimit"])
diffexp = current_app.data.diffexp_topN(
set1_filter, set2_filter, count, current_app.data.features["diffexp"]["interactiveLimit"]
)
except (ValueError, FilterError) as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except InteractiveError:
@@ -660,30 +564,27 @@ class DiffExpObsAPI(Resource):
class LayoutObsAPI(Resource):
@swagger.doc({
"summary": "Get the default layout for all observations.",
"tags": ["layout"],
"parameters": [],
"responses": {
"200": {
"description": "layout",
"examples": {
"application/json": {
"layout": {
"ndims": 2,
"coordinates": [
[0, 0.284483, 0.983744],
[1, 0.038844, 0.739444]
]
@swagger.doc(
{
"summary": "Get the default layout for all observations.",
"tags": ["layout"],
"parameters": [],
"responses": {
"200": {
"description": "layout",
"examples": {
"application/json": {
"layout": {
"ndims": 2,
"coordinates": [[0, 0.284_483, 0.983_744], [1, 0.038_844, 0.739_444]],
}
}
}
}
},
},
"400": {"description": "Data preparation error"},
},
"400": {
"description": "Data preparation error"
}
}
})
)
def get(self):
try:
layout = current_app.data.layout({})
server/app/scanpy_engine/diffexp.py
+5 -9
View File
@@ -1,4 +1,3 @@
import numpy as np
from scipy import sparse, stats
@@ -64,19 +63,19 @@ def diffexp_ttest(adata, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
sum_vn = vnA + vnB
# degrees of freedom for Welch's t-test
with np.errstate(divide='ignore', invalid='ignore'):
dof = sum_vn**2 / (vnA**2 / (nA - 1) + vnB**2 / (nB - 1))
with np.errstate(divide="ignore", invalid="ignore"):
dof = sum_vn ** 2 / (vnA ** 2 / (nA - 1) + vnB ** 2 / (nB - 1))
dof[np.isnan(dof)] = 1
# Welch's t-test score calculation
with np.errstate(divide='ignore', invalid='ignore'):
with np.errstate(divide="ignore", invalid="ignore"):
tscores = (meanA - meanB) / np.sqrt(sum_vn)
tscores[np.isnan(tscores)] = 0
# p-value
pvals = stats.t.sf(np.abs(tscores), dof) * 2
pvals_adj = pvals * adata._X.shape[1]
pvals_adj[pvals_adj > 1] = 1 # cap adjusted p-value at 1
pvals_adj[pvals_adj > 1] = 1 # cap adjusted p-value at 1
# logfoldchanges: log2(meanA / meanB)
logfoldchanges = np.log2(np.abs((meanA + 1e-9) / (meanB + 1e-9)))
@@ -106,8 +105,5 @@ def diffexp_ttest(adata, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
pvals_adj_top_n = pvals_adj[sort_order]
# varIndex, logfoldchange, pval, pval_adj
result = [[sort_order[i],
logfoldchanges_top_n[i],
pvals_top_n[i],
pvals_adj_top_n[i]] for i in range(top_n)]
result = [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n)]
return result
server/app/scanpy_engine/scanpy_engine.py
+53 -62
View File
@@ -22,7 +22,6 @@ Sort order for methods
class ScanpyEngine(CXGDriver):
def __init__(self, data, args):
super().__init__(data, args)
self._alias_annotation_names(Axis.OBS, args["obs_names"])
@@ -36,7 +35,7 @@ class ScanpyEngine(CXGDriver):
self._create_schema()
# TODO: temporary work-arounds
if args['nan_to_num']:
if args["nan_to_num"]:
self._IEEE754_special_values_workaround()
def _alias_annotation_names(self, axis, name):
@@ -61,8 +60,9 @@ class ScanpyEngine(CXGDriver):
if name not in df_axis.columns:
raise KeyError(f"Annotation name {name}, specified in --{ax_name}-name does not exist.")
if not df_axis[name].is_unique:
raise KeyError(f"Values in -{ax_name}-name must be unique. "
"Please prepare data to contain unique values.")
raise KeyError(
f"Values in -{ax_name}-name must be unique. " "Please prepare data to contain unique values."
)
# reset index to simple range; alias user-specified annotation to "name"
df_axis.reset_index(drop=True, inplace=True)
df_axis.rename(inplace=True, columns={name: "name"})
@@ -89,15 +89,8 @@ class ScanpyEngine(CXGDriver):
def _create_schema(self):
self.schema = {
"dataframe": {
"nObs": self.cell_count,
"nVar": self.gene_count,
"type": str(self.data.X.dtype)
},
"annotations": {
"obs": [],
"var": []
}
"dataframe": {"nObs": self.cell_count, "nVar": self.gene_count, "type": str(self.data.X.dtype)},
"annotations": {"obs": [], "var": []},
}
for ax in Axis:
curr_axis = getattr(self.data, str(ax))
@@ -129,32 +122,35 @@ class ScanpyEngine(CXGDriver):
try:
result = sc.read(data, cache=True)
except ValueError:
raise ScanpyFileError("File must be in the .h5ad format. Please read "
"https://github.com/theislab/scanpy_usage/blob/master/170505_seurat/info_h5ad.md to "
"learn more about this format. You may be able to convert your file into this format "
"using `cellxgene prepare`, please run `cellxgene prepare --help` for more "
"information.")
raise ScanpyFileError(
"File must be in the .h5ad format. Please read "
"https://github.com/theislab/scanpy_usage/blob/master/170505_seurat/info_h5ad.md to "
"learn more about this format. You may be able to convert your file into this format "
"using `cellxgene prepare`, please run `cellxgene prepare --help` for more "
"information."
)
except Exception as e:
raise ScanpyFileError(f"Error while loading file: {e}, File must be in the .h5ad format, please check "
f"that your input and try again.")
raise ScanpyFileError(
f"Error while loading file: {e}, File must be in the .h5ad format, please check "
f"that your input and try again."
)
return result
def _validate_data_types(self):
if self.data.X.dtype != "float32":
warnings.warn(f"Scanpy data matrix is in {self.data.X.dtype} format not float32. "
f"Precision may be truncated.")
warnings.warn(
f"Scanpy data matrix is in {self.data.X.dtype} format not float32. " f"Precision may be truncated."
)
for ax in Axis:
curr_axis = getattr(self.data, str(ax))
for ann in curr_axis:
datatype = curr_axis[ann].dtype
downcast_map = {"int64": "int32",
"uint32": "int32",
"uint64": "int32",
"float64": "float32",
}
downcast_map = {"int64": "int32", "uint32": "int32", "uint64": "int32", "float64": "float32"}
if datatype in downcast_map:
warnings.warn(f"Scanpy annotation {ax}:{ann} is in unsupported format: {datatype}. "
f"Data will be downcast to {downcast_map[datatype]}.")
warnings.warn(
f"Scanpy annotation {ax}:{ann} is in unsupported format: {datatype}. "
f"Data will be downcast to {downcast_map[datatype]}."
)
if isinstance(datatype, CategoricalDtype):
category_num = len(curr_axis[ann].dtype.categories)
if category_num > 500 and category_num > self.max_category_items:
@@ -162,7 +158,8 @@ class ScanpyEngine(CXGDriver):
f"{str(ax).title()} annotation '{ann}' has {category_num} categories, this may be "
f"cumbersome or slow to display. We recommend setting the "
f"--max-category-items option to 500, this will hide categorical "
f"annotations with more than 500 categories in the UI")
f"annotations with more than 500 categories in the UI"
)
def _validate_data_calculations(self):
layout_key = f"X_{self.layout_method}"
@@ -174,7 +171,8 @@ class ScanpyEngine(CXGDriver):
f" layout may have been computed. The requested layout must be pre-calculated and saved "
f"back in the h5ad file. You can run "
f"`cellxgene prepare --layout {self.layout_method} <datafile>` "
f"to solve this problem. ")
f"to solve this problem. "
)
def _IEEE754_special_values_workaround(self):
"""
@@ -196,7 +194,7 @@ class ScanpyEngine(CXGDriver):
curr_axis = getattr(self.data, str(ax))
for ann in curr_axis:
dtype = curr_axis[ann].dtype
if dtype.kind == 'f':
if dtype.kind == "f":
finite_idx = np.isfinite(curr_axis[ann])
if not finite_idx.all():
curr_axis.loc[np.isnan(curr_axis[ann]), ann] = 0
@@ -233,8 +231,7 @@ class ScanpyEngine(CXGDriver):
if non_finite_X_found:
warnings.warn(
"Dataframe X contains floating point NaN or Infinities. "
"These will be converted to finite values."
"Dataframe X contains floating point NaN or Infinities. " "These will be converted to finite values."
)
def filter_dataframe(self, filter):
@@ -256,7 +253,7 @@ class ScanpyEngine(CXGDriver):
@staticmethod
def _annotation_filter_to_mask(filter, d_axis, count):
mask = np.ones((count, ), dtype=bool)
mask = np.ones((count,), dtype=bool)
for v in filter:
if d_axis[v["name"]].dtype.name in ["boolean", "category", "object"]:
key_idx = np.in1d(getattr(d_axis, v["name"]), v["values"])
@@ -274,24 +271,23 @@ class ScanpyEngine(CXGDriver):
@staticmethod
def _index_filter_to_mask(filter, count):
mask = np.zeros((count, ), dtype=bool)
mask = np.zeros((count,), dtype=bool)
for i in filter:
if type(i) == list:
mask[i[0]:i[1]] = True
mask[i[0] : i[1]] = True
else:
mask[i] = True
return mask
@staticmethod
def _axis_filter_to_mask(filter, d_axis, count):
mask = np.ones((count, ), dtype=bool)
mask = np.ones((count,), dtype=bool)
if "index" in filter:
mask = np.logical_and(mask, ScanpyEngine._index_filter_to_mask(filter["index"], count))
if "annotation_value" in filter:
mask = np.logical_and(mask,
ScanpyEngine._annotation_filter_to_mask(filter["annotation_value"],
d_axis,
count))
mask = np.logical_and(
mask, ScanpyEngine._annotation_filter_to_mask(filter["annotation_value"], d_axis, count)
)
return mask
def _filter_to_mask(self, filter, use_slices=True):
@@ -321,8 +317,9 @@ class ScanpyEngine(CXGDriver):
https://docs.scipy.org/doc/scipy/reference/sparse.html
"""
prefer_row_access = sparse.isspmatrix_csr(data._X) or sparse.isspmatrix_lil(data._X) \
or sparse.isspmatrix_bsr(data._X)
prefer_row_access = (
sparse.isspmatrix_csr(data._X) or sparse.isspmatrix_lil(data._X) or sparse.isspmatrix_bsr(data._X)
)
if prefer_row_access:
# Row-major slicing
if obs_selector is not None:
@@ -355,18 +352,12 @@ class ScanpyEngine(CXGDriver):
obs = self.data.obs[obs_selector]
if not fields:
fields = obs.columns.tolist()
result = {
"names": fields,
"data": DataFrame(obs[fields]).to_records(index=True).tolist()
}
result = {"names": fields, "data": DataFrame(obs[fields]).to_records(index=True).tolist()}
else:
var = self.data.var[var_selector]
if not fields:
fields = var.columns.tolist()
result = {
"names": fields,
"data": DataFrame(var[fields]).to_records(index=True).tolist()
}
result = {"names": fields, "data": DataFrame(var[fields]).to_records(index=True).tolist()}
return result
def data_frame(self, filter, axis):
@@ -391,12 +382,12 @@ class ScanpyEngine(CXGDriver):
if axis == Axis.OBS:
result = {
"var": var_index_sliced.tolist(),
"obs": DataFrame(_X, index=obs_index_sliced).to_records(index=True).tolist()
"obs": DataFrame(_X, index=obs_index_sliced).to_records(index=True).tolist(),
}
else:
result = {
"obs": obs_index_sliced.tolist(),
"var": DataFrame(_X.T, index=var_index_sliced).to_records(index=True).tolist()
"var": DataFrame(_X.T, index=var_index_sliced).to_records(index=True).tolist(),
}
return result
@@ -435,11 +426,11 @@ class ScanpyEngine(CXGDriver):
try:
df_layout = df.obsm[f"X_{self.layout_method}"]
except ValueError as e:
raise PrepareError(f"Layout has not been calculated using {self.layout_method}, "
f"please prepare your datafile and relaunch cellxgene") from e
normalized_layout = DataFrame((df_layout - df_layout.min()) / (df_layout.max() - df_layout.min()),
index=df.obs.index)
return {
"ndims": normalized_layout.shape[1],
"coordinates": normalized_layout.to_records(index=True).tolist()
}
raise PrepareError(
f"Layout has not been calculated using {self.layout_method}, "
f"please prepare your datafile and relaunch cellxgene"
) from e
normalized_layout = DataFrame(
(df_layout - df_layout.min()) / (df_layout.max() - df_layout.min()), index=df.obs.index
)
return {"ndims": normalized_layout.shape[1], "coordinates": normalized_layout.to_records(index=True).tolist()}
server/app/util/filter.py
-1
View File
@@ -7,7 +7,6 @@ from server.app.util.constants import Axis
class QueryStringError(Exception):
def __init__(self, key, message):
self.key = key
self.message = message
server/app/util/models.py
+7 -38
View File
@@ -5,24 +5,13 @@ class AnnotationModel(Schema):
type = "object"
description = "Filter by annotation key: value"
properties = {
"name": {
"type": "string"
},
"name": {"type": "string"},
# TODO update to OpenAPI v3.0 when a library is available that supports it
# Unfortunately 2.0 doesn't have a way to have a schema that accepts multiple types
# Overloading the type key with a list seems to work ok and makes it to the page
"values": {
"type": "array",
"items": {
"type": ["float32", "string", "int32", "bool"]
}
},
"min": {
"type": ["int32", "float32"],
},
"max": {
"type": ["int32", "float32"],
}
"values": {"type": "array", "items": {"type": ["float32", "string", "int32", "bool"]}},
"min": {"type": ["int32", "float32"]},
"max": {"type": ["int32", "float32"]},
}
required = ["name"]
@@ -30,36 +19,16 @@ class AnnotationModel(Schema):
class IndexModel(Schema):
type = "object"
description = "Filter by index of observation/variable ex. [0, 5, 15]"
properties = {
"index": {
"type": "array",
"items": {
"format": "int32",
"type": "integer"
}
}
}
properties = {"index": {"type": "array", "items": {"format": "int32", "type": "integer"}}}
class AxisModel(Schema):
type = "object"
description = "Axis of data -- obs or var"
properties = {
"index": IndexModel,
"annotation_value": AnnotationModel.array()
}
properties = {"index": IndexModel, "annotation_value": AnnotationModel.array()}
class FilterModel(Schema):
type = "object"
description = "Complex filter"
properties = {
"filter": {
"type": "object",
"properties": {
"obs": AxisModel,
"var": AxisModel
}
}
}
properties = {"filter": {"type": "object", "properties": {"obs": AxisModel, "var": AxisModel}}}
server/app/util/utils.py
+4 -3
View File
@@ -15,7 +15,7 @@ class Float32JSONEncoder(json.JSONEncoder):
if it runs into non-finite floating point values which are unsupported by
standard JSON.
"""
kwargs['allow_nan'] = False
kwargs["allow_nan"] = False
super().__init__(*args, **kwargs)
def default(self, obj):
@@ -30,8 +30,9 @@ def custom_format_warning(msg, *args, **kwargs):
return f"[cellxgene] Warning: {msg} \n"
def get_mime_type(default="application/json", acceptable_types=["application/json", "text/csv"], query_param=None,
header=None):
def get_mime_type(
default="application/json", acceptable_types=["application/json", "text/csv"], query_param=None, header=None
):
mime_type = default
if query_param:
if query_param in acceptable_types:
server/app/web/webapp.py
+1 -3
View File
@@ -1,7 +1,5 @@
import os
from flask import (
Blueprint, render_template, send_from_directory, current_app
)
from flask import Blueprint, render_template, send_from_directory, current_app
bp = Blueprint("webapp", __name__, template_folder="templates")