fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-10-10 01:30:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

Black -- formatter for python (#508)

Browse Source 
* Add black

* use black to format code

* Black version
This commit is contained in:
Charlotte Weaver
2018-12-12 09:44:47 -08:00
committed by GitHub
parent a847951658
commit 83154577e4
18 changed files with 636 additions and 791 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
+2 -2
View File
@@ -14,8 +14,8 @@ install:
- docker build . - docker build .
script: script:
- set -eo pipefail - set -eo pipefail
- flake8 server/app/ - flake8 server
- flake8 server/cli/ - black --check
- npm run --prefix client/ build - npm run --prefix client/ build
- npm run --prefix client/ test - npm run --prefix client/ test
- pytest -s server/test - pytest -s server/test
server/__main__.py
+5 -2
View File
@@ -2,11 +2,14 @@
if __package__ is None: if __package__ is None:
import sys import sys
from pathlib import Path from pathlib import Path
PKG_PATH = Path(__file__).parent PKG_PATH = Path(__file__).parent
sys.path.insert(0, str(PKG_PATH.parent)) sys.path.insert(0, str(PKG_PATH.parent))
import server import server # noqa F401
__package__ = PKG_PATH.name __package__ = PKG_PATH.name
# Main thing # Main thing
from .cli.cli import cli from .cli.cli import cli # noqa F402
cli() cli()
server/app/app.py
+10 -6
View File
@@ -14,16 +14,14 @@ REACTIVE_LIMIT = 1_000_000
app = Flask(__name__, static_folder="web/static") app = Flask(__name__, static_folder="web/static")
app.json_encoder = Float32JSONEncoder app.json_encoder = Float32JSONEncoder
cache = Cache(app, config={"CACHE_TYPE": "simple", "CACHE_DEFAULT_TIMEOUT": 860000}) cache = Cache(app, config={"CACHE_TYPE": "simple", "CACHE_DEFAULT_TIMEOUT": 860_000})
Compress(app) Compress(app)
CORS(app) CORS(app)
# Config # Config
SECRET_KEY = os.environ.get("CXG_SECRET_KEY", default="SparkleAndShine") SECRET_KEY = os.environ.get("CXG_SECRET_KEY", default="SparkleAndShine")
app.config.update( app.config.update(SECRET_KEY=SECRET_KEY)
SECRET_KEY=SECRET_KEY,
)
# Application Data # Application Data
data = None data = None
@@ -36,7 +34,13 @@ docs.append(resources.get_swagger_doc())
app.register_blueprint(webapp.bp) app.register_blueprint(webapp.bp)
app.register_blueprint(resources.blueprint) app.register_blueprint(resources.blueprint)
app.register_blueprint( app.register_blueprint(
get_swagger_blueprint(docs, "/api/swagger", produces=["application/json"], title="cellxgene rest api", get_swagger_blueprint(
description="An API connecting ExpressionMatrix2 clustering algorithm to cellxgene")) docs,
"/api/swagger",
produces=["application/json"],
title="cellxgene rest api",
description="An API connecting ExpressionMatrix2 clustering algorithm to cellxgene",
)
)
app.add_url_rule("/", endpoint="index") app.add_url_rule("/", endpoint="index")
server/app/driver/driver.py
+2 -6
View File
@@ -11,7 +11,6 @@ Sort order for methods
class CXGDriver(metaclass=ABCMeta): class CXGDriver(metaclass=ABCMeta):
def __init__(self, data, args): def __init__(self, data, args):
self.data = self._load_data(data) self.data = self._load_data(data)
self.layout_method = args["layout"] self.layout_method = args["layout"]
@@ -24,11 +23,8 @@ class CXGDriver(metaclass=ABCMeta):
def features(self): def features(self):
features = { features = {
"cluster": {"available": False}, "cluster": {"available": False},
"layout": { "layout": {"obs": {"available": False}, "var": {"available": False}},
"obs": {"available": False}, "diffexp": {"available": False},
"var": {"available": False},
},
"diffexp": {"available": False}
} }
# TODO - Interactive limit should be generated from the actual available methods see GH issue #94 # TODO - Interactive limit should be generated from the actual available methods see GH issue #94
if self.layout_method: if self.layout_method:
server/app/rest_api/rest.py
+327 -426
View File
@@ -2,9 +2,7 @@ from http import HTTPStatus
import pkg_resources import pkg_resources
import warnings import warnings
from flask import ( from flask import Blueprint, current_app, jsonify, make_response, request
Blueprint, current_app, jsonify, make_response, request
)
from flask_restful_swagger_2 import Api, swagger, Resource from flask_restful_swagger_2 import Api, swagger, Resource
from werkzeug.datastructures import ImmutableMultiDict from werkzeug.datastructures import ImmutableMultiDict
@@ -23,83 +21,78 @@ Sort order for routes
class SchemaAPI(Resource): class SchemaAPI(Resource):
@swagger.doc({ @swagger.doc(
"summary": "get schema for dataframe and annotations", {
"tags": ["initialize"], "summary": "get schema for dataframe and annotations",
"parameters": [], "tags": ["initialize"],
"responses": { "parameters": [],
"200": { "responses": {
"description": "schema", "200": {
"examples": { "description": "schema",
"application/json": { "examples": {
"schema": { "application/json": {
"dataframe": { "schema": {
"nObs": 383, "dataframe": {"nObs": 383, "nVar": 19944, "type": "float32"},
"nVar": 19944, "annotations": {
"type": "float32" "obs": [
}, {"name": "name", "type": "string"},
"annotations": { {"name": "tissue_type", "type": "string"},
"obs": [ {"name": "num_reads", "type": "int32"},
{"name": "name", "type": "string"}, {"name": "sample_name", "type": "string"},
{"name": "tissue_type", "type": "string"}, {
{"name": "num_reads", "type": "int32"}, "name": "clusters",
{"name": "sample_name", "type": "string"}, "type": "categorical",
{ "categories": [99, 1, "unknown cluster"],
"name": "clusters", },
"type": "categorical", {"name": "QScore", "type": "float32"},
"categories": [99, 1, "unknown cluster"] ],
}, "var": [{"name": "name", "type": "string"}, {"name": "gene", "type": "string"}],
{"name": "QScore", "type": "float32"} },
],
"var": [
{"name": "name", "type": "string"},
{"name": "gene", "type": "string"}
]
} }
} }
} },
} }
} },
} }
)
})
def get(self): def get(self):
return make_response(jsonify({"schema": current_app.data.schema}), HTTPStatus.OK) return make_response(jsonify({"schema": current_app.data.schema}), HTTPStatus.OK)
class ConfigAPI(Resource): class ConfigAPI(Resource):
@swagger.doc({ @swagger.doc(
"summary": "Configuration information to assist in front-end adaptation" {
" to underlying engine, available functionality, interactive time limits, etc", "summary": "Configuration information to assist in front-end adaptation"
"tags": ["initialize"], " to underlying engine, available functionality, interactive time limits, etc",
"parameters": [], "tags": ["initialize"],
"responses": { "parameters": [],
"200": { "responses": {
"description": "schema", "200": {
"examples": { "description": "schema",
"application/json": { "examples": {
"config": { "application/json": {
"features": [ "config": {
{"method": "POST", "path": "/cluster/", "available": False}, "features": [
{ {"method": "POST", "path": "/cluster/", "available": False},
"method": "POST", {
"path": "/layout/obs", "method": "POST",
"available": True, "path": "/layout/obs",
"interactiveLimit": 10000 "available": True,
"interactiveLimit": 10000,
},
{"method": "POST", "path": "/layout/var", "available": False},
],
"displayNames": {
"engine": "ScanPy version 1.33",
"dataset": "/home/joe/mouse/blorth.csv",
}, },
{"method": "POST", "path": "/layout/var", "available": False} }
],
"displayNames": {
"engine": "ScanPy version 1.33",
"dataset": "/home/joe/mouse/blorth.csv"
},
} }
} },
} }
} },
} }
}) )
def get(self): def get(self):
config = { config = {
"config": { "config": {
@@ -111,50 +104,48 @@ class ConfigAPI(Resource):
], ],
"displayNames": { "displayNames": {
"engine": f"cellxgene Scanpy engine version {pkg_resources.get_distribution('cellxgene').version}", "engine": f"cellxgene Scanpy engine version {pkg_resources.get_distribution('cellxgene').version}",
"dataset": current_app.config["DATASET_TITLE"] "dataset": current_app.config["DATASET_TITLE"],
}, },
"parameters": { "parameters": {"max_category_items": current_app.data.max_category_items},
"max_category_items": current_app.data.max_category_items
}
} }
} }
return make_response(jsonify(config), HTTPStatus.OK) return make_response(jsonify(config), HTTPStatus.OK)
class AnnotationsObsAPI(Resource): class AnnotationsObsAPI(Resource):
@swagger.doc({ @swagger.doc(
"summary": "Fetch annotations (metadata) for all observations.", {
"tags": ["annotations"], "summary": "Fetch annotations (metadata) for all observations.",
"parameters": [{ "tags": ["annotations"],
"in": "query", "parameters": [
"name": "annotation-name", {
"type": "string", "in": "query",
"description": "list of 1 or more annotation names" "name": "annotation-name",
}], "type": "string",
"responses": { "description": "list of 1 or more annotation names",
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": [
"tissue_type", "sex", "num_reads", "clusters"
],
"data": [
[0, "lung", "F", 39844, 99],
[1, "heart", "M", 83, 1],
[49, "spleen", None, 2, "unknown cluster"],
]
}
} }
],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": ["tissue_type", "sex", "num_reads", "clusters"],
"data": [
[0, "lung", "F", 39844, 99],
[1, "heart", "M", 83, 1],
[49, "spleen", None, 2, "unknown cluster"],
],
}
},
},
"400": {
"description": "one or more of the annotation-name identifiers were not associated with an "
"annotation name"
},
}, },
"400": {
"description": "one or more of the annotation-name identifiers were not associated with an "
"annotation name"
}
} }
}) )
def get(self): def get(self):
fields = request.args.getlist("annotation-name", None) fields = request.args.getlist("annotation-name", None)
try: try:
@@ -168,47 +159,40 @@ class AnnotationsObsAPI(Resource):
warnings.warn(JSON_NaN_to_num_warning_msg) warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
@swagger.doc({ @swagger.doc(
"summary": "Fetch annotations (metadata) for filtered subset of observations.", {
"tags": ["annotations"], "summary": "Fetch annotations (metadata) for filtered subset of observations.",
"parameters": [ "tags": ["annotations"],
{ "parameters": [
"in": "query", {
"name": "annotation-name", "in": "query",
"type": "string", "name": "annotation-name",
"description": "list of 1 or more annotation names" "type": "string",
"description": "list of 1 or more annotation names",
},
{"name": "filter", "description": "Complex Filter", "in": "body", "schema": FilterModel},
],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": ["tissue_type", "sex", "num_reads", "clusters"],
"data": [
[0, "lung", "F", 39844, 99],
[1, "heart", "M", 83, 1],
[49, "spleen", None, 2, "unknown cluster"],
],
}
},
},
"400": {
"description": "malformed filter or one or more of the annotation-name identifiers were"
"not associated with an annotation name"
},
}, },
{
"name": "filter",
"description": "Complex Filter",
"in": "body",
"schema": FilterModel
}
],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": [
"tissue_type", "sex", "num_reads", "clusters"
],
"data": [
[0, "lung", "F", 39844, 99],
[1, "heart", "M", 83, 1],
[49, "spleen", None, 2, "unknown cluster"],
]
}
}
},
"400": {
"description": "malformed filter or one or more of the annotation-name identifiers were"
"not associated with an annotation name"
}
} }
}) )
def put(self): def put(self):
fields = request.args.getlist("annotation-name", None) fields = request.args.getlist("annotation-name", None)
try: try:
@@ -226,38 +210,35 @@ class AnnotationsObsAPI(Resource):
class AnnotationsVarAPI(Resource): class AnnotationsVarAPI(Resource):
@swagger.doc({ @swagger.doc(
"summary": "Fetch annotations (metadata) for all variables.", {
"tags": ["annotations"], "summary": "Fetch annotations (metadata) for all variables.",
"parameters": [{ "tags": ["annotations"],
"in": "query", "parameters": [
"name": "annotation-name", {
"type": "string", "in": "query",
"description": "list of 1 or more annotation names" "name": "annotation-name",
}], "type": "string",
"responses": { "description": "list of 1 or more annotation names",
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": [
"name", "category"
],
"data": [
[0, "ATAD3C", 1],
[1, "RER1", None],
[49, "S100B", 6]
]
}
} }
],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": ["name", "category"],
"data": [[0, "ATAD3C", 1], [1, "RER1", None], [49, "S100B", 6]],
}
},
},
"400": {
"description": "one or more of the annotation-name identifiers were not associated with an"
" annotation name"
},
}, },
"400": {
"description": "one or more of the annotation-name identifiers were not associated with an"
" annotation name"
}
} }
}) )
def get(self): def get(self):
fields = request.args.getlist("annotation-name", None) fields = request.args.getlist("annotation-name", None)
try: try:
@@ -271,45 +252,36 @@ class AnnotationsVarAPI(Resource):
warnings.warn(JSON_NaN_to_num_warning_msg) warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
@swagger.doc({ @swagger.doc(
"summary": "Fetch annotations (metadata) for filtered subset of variables.", {
"tags": ["annotations"], "summary": "Fetch annotations (metadata) for filtered subset of variables.",
"parameters": [ "tags": ["annotations"],
{ "parameters": [
"in": "query", {
"name": "annotation-name", "in": "query",
"type": "string", "name": "annotation-name",
"description": "list of 1 or more annotation names" "type": "string",
"description": "list of 1 or more annotation names",
},
{"name": "filter", "description": "Complex Filter", "in": "body", "schema": FilterModel},
],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": ["name", "category"],
"data": [[0, "ATAD3C", 1], [1, "RER1", None], [49, "S100B", 6]],
}
},
},
"400": {
"description": "malformed filter or one or more of the annotation-name identifiers were"
"not associated with an annotation name"
},
}, },
{
"name": "filter",
"description": "Complex Filter",
"in": "body",
"schema": FilterModel
}
],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": [
"name", "category"
],
"data": [
[0, "ATAD3C", 1],
[1, "RER1", None],
[49, "S100B", 6]
]
}
}
},
"400": {
"description": "malformed filter or one or more of the annotation-name identifiers were"
"not associated with an annotation name"
}
} }
}) )
def put(self): def put(self):
fields = request.args.getlist("annotation-name", None) fields = request.args.getlist("annotation-name", None)
try: try:
@@ -327,57 +299,39 @@ class AnnotationsVarAPI(Resource):
class DataObsAPI(Resource): class DataObsAPI(Resource):
@swagger.doc({ @swagger.doc(
"summary": "Get data (expression values) from the dataframe.", {
"tags": ["data"], "summary": "Get data (expression values) from the dataframe.",
"parameters": [ "tags": ["data"],
{ "parameters": [
"in": "query", {"in": "query", "name": "filter", "type": "string", "description": "axis:key:value"},
"name": "filter", {"in": "query", "name": "accept-type", "type": "string", "description": "MIME type"},
"type": "string", ],
"description": "axis:key:value" "responses": {
}, "200": {
{ "description": "expression",
"in": "query", "examples": {"application/json": {"var": [0, 20000], "obs": [[1, 39483, 3902, 203, 0, 0, 28]]}},
"name": "accept-type", },
"type": "string", "400": {"description": "Malformed filter"},
"description": "MIME type" "406": {"description": "Unacceptable MIME type"},
},
],
"responses": {
"200": {
"description": "expression",
"examples": {
"application/json": {
"var": [0, 20000],
"obs": [
[1, 39483, 3902, 203, 0, 0, 28]
]
}
}
},
"400": {
"description": "Malformed filter"
},
"406": {
"description": "Unacceptable MIME type"
}, },
} }
}) )
def get(self): def get(self):
accept_type = request.args.get("accept-type", None) accept_type = request.args.get("accept-type", None)
# request.args is immutable # request.args is immutable
args = request.args.copy() args = request.args.copy()
args.pop("accept-type", None) args.pop("accept-type", None)
try: try:
filter_ = parse_filter(ImmutableMultiDict(args), current_app.data.schema['annotations']) filter_ = parse_filter(ImmutableMultiDict(args), current_app.data.schema["annotations"])
except QueryStringError as e: except QueryStringError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST) return make_response(e.message, HTTPStatus.BAD_REQUEST)
# TODO support CSV # TODO support CSV
try: try:
# TODO store mime_type when more than one is supported # TODO store mime_type when more than one is supported
get_mime_type(acceptable_types=["application/json"], query_param=accept_type, get_mime_type(
header=request.accept_mimetypes) acceptable_types=["application/json"], query_param=accept_type, header=request.accept_mimetypes
)
except MimeTypeError as e: except MimeTypeError as e:
return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE) return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
try: try:
@@ -389,37 +343,21 @@ class DataObsAPI(Resource):
warnings.warn(JSON_NaN_to_num_warning_msg) warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
@swagger.doc({ @swagger.doc(
"summary": "Get data (expression values) from the dataframe.", {
"tags": ["data"], "summary": "Get data (expression values) from the dataframe.",
"parameters": [ "tags": ["data"],
{ "parameters": [{"name": "filter", "description": "Complex Filter", "in": "body", "schema": FilterModel}],
'name': 'filter', "responses": {
'description': 'Complex Filter', "200": {
'in': 'body', "description": "expression",
'schema': FilterModel "examples": {"application/json": {"var": [0, 20000], "obs": [[1, 39483, 3902, 203, 0, 0, 28]]}},
} },
], "400": {"description": "Malformed filter"},
"responses": { "406": {"description": "Unacceptable MIME type"},
"200": {
"description": "expression",
"examples": {
"application/json": {
"var": [0, 20000],
"obs": [
[1, 39483, 3902, 203, 0, 0, 28]
]
}
}
},
"400": {
"description": "Malformed filter"
},
"406": {
"description": "Unacceptable MIME type"
}, },
} }
}) )
def put(self): def put(self):
if not request.accept_mimetypes.best_match(["application/json", "text/csv"]): if not request.accept_mimetypes.best_match(["application/json", "text/csv"]):
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
@@ -428,8 +366,9 @@ class DataObsAPI(Resource):
except MimeTypeError as e: except MimeTypeError as e:
return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE) return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
try: try:
return make_response((jsonify(current_app.data.data_frame(request.get_json()["filter"], axis=Axis.OBS))), return make_response(
HTTPStatus.OK) (jsonify(current_app.data.data_frame(request.get_json()["filter"], axis=Axis.OBS))), HTTPStatus.OK
)
except FilterError as e: except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST) return make_response(e.message, HTTPStatus.BAD_REQUEST)
except ValueError as e: except ValueError as e:
@@ -439,55 +378,37 @@ class DataObsAPI(Resource):
class DataVarAPI(Resource): class DataVarAPI(Resource):
@swagger.doc({ @swagger.doc(
"summary": "Get data (expression values) from the dataframe.", {
"tags": ["data"], "summary": "Get data (expression values) from the dataframe.",
"parameters": [ "tags": ["data"],
{ "parameters": [
"in": "query", {"in": "query", "name": "filter", "type": "string", "description": "axis:key:value"},
"name": "filter", {"in": "query", "name": "accept-type", "type": "string", "description": "MIME type"},
"type": "string", ],
"description": "axis:key:value" "responses": {
}, "200": {
{ "description": "expression",
"in": "query", "examples": {"application/json": {"obs": [0, 20000], "var": [[1, 39483, 3902, 203, 0, 0, 28]]}},
"name": "accept-type", },
"type": "string", "400": {"description": "Malformed filter"},
"description": "MIME type" "406": {"description": "Unacceptable MIME type"},
},
],
"responses": {
"200": {
"description": "expression",
"examples": {
"application/json": {
"obs": [0, 20000],
"var": [
[1, 39483, 3902, 203, 0, 0, 28]
]
}
}
},
"400": {
"description": "Malformed filter"
},
"406": {
"description": "Unacceptable MIME type"
}, },
} }
}) )
def get(self): def get(self):
accept_type = request.args.get("accept-type", None) accept_type = request.args.get("accept-type", None)
# request.args is immutable # request.args is immutable
args = request.args.copy() args = request.args.copy()
args.pop("accept-type", None) args.pop("accept-type", None)
try: try:
filter_ = parse_filter(ImmutableMultiDict(args), current_app.data.schema['annotations']) filter_ = parse_filter(ImmutableMultiDict(args), current_app.data.schema["annotations"])
except QueryStringError as e: except QueryStringError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST) return make_response(e.message, HTTPStatus.BAD_REQUEST)
try: try:
get_mime_type(acceptable_types=["application/json"], query_param=accept_type, get_mime_type(
header=request.accept_mimetypes) acceptable_types=["application/json"], query_param=accept_type, header=request.accept_mimetypes
)
except MimeTypeError as e: except MimeTypeError as e:
return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE) return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
try: try:
@@ -499,37 +420,21 @@ class DataVarAPI(Resource):
warnings.warn(JSON_NaN_to_num_warning_msg) warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
@swagger.doc({ @swagger.doc(
"summary": "Get data (expression values) from the dataframe.", {
"tags": ["data"], "summary": "Get data (expression values) from the dataframe.",
"parameters": [ "tags": ["data"],
{ "parameters": [{"name": "filter", "description": "Complex Filter", "in": "body", "schema": FilterModel}],
'name': 'filter', "responses": {
'description': 'Complex Filter', "200": {
'in': 'body', "description": "expression",
'schema': FilterModel "examples": {"application/json": {"obs": [0, 20000], "var": [[1, 39483, 3902, 203, 0, 0, 28]]}},
} },
], "400": {"description": "Malformed filter"},
"responses": { "406": {"description": "Unacceptable MIME type"},
"200": {
"description": "expression",
"examples": {
"application/json": {
"obs": [0, 20000],
"var": [
[1, 39483, 3902, 203, 0, 0, 28]
]
}
}
},
"400": {
"description": "Malformed filter"
},
"406": {
"description": "Unacceptable MIME type"
}, },
} }
}) )
def put(self): def put(self):
if not request.accept_mimetypes.best_match(["application/json", "text/csv"]): if not request.accept_mimetypes.best_match(["application/json", "text/csv"]):
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
@@ -539,8 +444,9 @@ class DataVarAPI(Resource):
except MimeTypeError as e: except MimeTypeError as e:
return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE) return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
try: try:
return make_response((jsonify(current_app.data.data_frame(request.get_json()["filter"], axis=Axis.VAR))), return make_response(
HTTPStatus.OK) (jsonify(current_app.data.data_frame(request.get_json()["filter"], axis=Axis.VAR))), HTTPStatus.OK
)
except FilterError as e: except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST) return make_response(e.message, HTTPStatus.BAD_REQUEST)
except ValueError as e: except ValueError as e:
@@ -550,67 +456,64 @@ class DataVarAPI(Resource):
class DiffExpObsAPI(Resource): class DiffExpObsAPI(Resource):
@swagger.doc({ @swagger.doc(
"summary": "Generate differential expression (DE) statistics for two specified subsets of data, " {
"as indicated by the two provided observation complex filters", "summary": "Generate differential expression (DE) statistics for two specified subsets of data, "
"tags": ["diffexp"], "as indicated by the two provided observation complex filters",
# TODO sort out params "tags": ["diffexp"],
# "parameters": [ # TODO sort out params
# # { # "parameters": [
# # "in": "body", # # {
# # "name": "mode", # # "in": "body",
# # "type": "string", # # "name": "mode",
# # "required": True, # # "type": "string",
# # "description": "topN or varFilter" # # "required": True,
# # }, # # "description": "topN or varFilter"
# { # # },
# "in": "query", # {
# "name": "count", # "in": "query",
# "type": "int32", # "name": "count",
# "description": "TopN mode: how many vars to return" # "type": "int32",
# }, # "description": "TopN mode: how many vars to return"
# { # },
# "in": "body", # {
# "name": "varFilter", # "in": "body",
# "schema": FilterModel, # "name": "varFilter",
# "description": "varFilter: Complex filter, only var for which vars to return" # "schema": FilterModel,
# }, # "description": "varFilter: Complex filter, only var for which vars to return"
# { # },
# "in": "body", # {
# "name": "set1", # "in": "body",
# "schema": FilterModel, # "name": "set1",
# "required": True, # "schema": FilterModel,
# "description": "Complex filter, only obs - observations in set1" # "required": True,
# }, # "description": "Complex filter, only obs - observations in set1"
# { # },
# "in": "body", # {
# "name": "set2", # "in": "body",
# "schema": FilterModel, # "name": "set2",
# "description": "Complex filter, only obs - observations in set2. If not included, inverse of set1." # "schema": FilterModel,
# }, # "description": "Complex filter, only obs - observations in set2.
# ], # If not included, inverse of set1."
"responses": { # },
"200": { # ],
"description": "Statistics are encoded as an array of arrays, with fields ordered as: " "responses": {
"varIndex, logfoldchange, pVal, pValAdj", "200": {
"examples": { "description": "Statistics are encoded as an array of arrays, with fields ordered as: "
"application/json": [ "varIndex, logfoldchange, pVal, pValAdj",
[328, -2.569489, 2.655706e-63, 3.642036e-57], "examples": {
[1250, -2.569489, 2.655706e-63, 3.642036e-57], "application/json": [
] [328, -2.569_489, 2.655_706e-63, 3.642_036e-57],
} [1250, -2.569_489, 2.655_706e-63, 3.642_036e-57],
]
},
},
"400": {"description": "malformed filter"},
"403": {"description": "non-interactive request"},
"501": {"description": "diffexp is not implemented"},
}, },
"400": {
"description": "malformed filter"
},
"403": {
"description": "non-interactive request"
},
"501": {
"description": "diffexp is not implemented"
}
} }
}) )
def post(self): def post(self):
args = request.get_json() args = request.get_json()
# confirm mode is present and legal # confirm mode is present and legal
@@ -645,8 +548,9 @@ class DiffExpObsAPI(Resource):
# mode=topN # mode=topN
count = args.get("count", None) count = args.get("count", None)
try: try:
diffexp = current_app.data.diffexp_topN(set1_filter, set2_filter, count, diffexp = current_app.data.diffexp_topN(
current_app.data.features["diffexp"]["interactiveLimit"]) set1_filter, set2_filter, count, current_app.data.features["diffexp"]["interactiveLimit"]
)
except (ValueError, FilterError) as e: except (ValueError, FilterError) as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST) return make_response(e.message, HTTPStatus.BAD_REQUEST)
except InteractiveError: except InteractiveError:
@@ -660,30 +564,27 @@ class DiffExpObsAPI(Resource):
class LayoutObsAPI(Resource): class LayoutObsAPI(Resource):
@swagger.doc({ @swagger.doc(
"summary": "Get the default layout for all observations.", {
"tags": ["layout"], "summary": "Get the default layout for all observations.",
"parameters": [], "tags": ["layout"],
"responses": { "parameters": [],
"200": { "responses": {
"description": "layout", "200": {
"examples": { "description": "layout",
"application/json": { "examples": {
"layout": { "application/json": {
"ndims": 2, "layout": {
"coordinates": [ "ndims": 2,
[0, 0.284483, 0.983744], "coordinates": [[0, 0.284_483, 0.983_744], [1, 0.038_844, 0.739_444]],
[1, 0.038844, 0.739444] }
]
} }
} },
} },
"400": {"description": "Data preparation error"},
}, },
"400": {
"description": "Data preparation error"
}
} }
}) )
def get(self): def get(self):
try: try:
layout = current_app.data.layout({}) layout = current_app.data.layout({})
server/app/scanpy_engine/diffexp.py
+5 -9
View File
@@ -1,4 +1,3 @@
import numpy as np import numpy as np
from scipy import sparse, stats from scipy import sparse, stats
@@ -64,19 +63,19 @@ def diffexp_ttest(adata, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
sum_vn = vnA + vnB sum_vn = vnA + vnB
# degrees of freedom for Welch's t-test # degrees of freedom for Welch's t-test
with np.errstate(divide='ignore', invalid='ignore'): with np.errstate(divide="ignore", invalid="ignore"):
dof = sum_vn**2 / (vnA**2 / (nA - 1) + vnB**2 / (nB - 1)) dof = sum_vn ** 2 / (vnA ** 2 / (nA - 1) + vnB ** 2 / (nB - 1))
dof[np.isnan(dof)] = 1 dof[np.isnan(dof)] = 1
# Welch's t-test score calculation # Welch's t-test score calculation
with np.errstate(divide='ignore', invalid='ignore'): with np.errstate(divide="ignore", invalid="ignore"):
tscores = (meanA - meanB) / np.sqrt(sum_vn) tscores = (meanA - meanB) / np.sqrt(sum_vn)
tscores[np.isnan(tscores)] = 0 tscores[np.isnan(tscores)] = 0
# p-value # p-value
pvals = stats.t.sf(np.abs(tscores), dof) * 2 pvals = stats.t.sf(np.abs(tscores), dof) * 2
pvals_adj = pvals * adata._X.shape[1] pvals_adj = pvals * adata._X.shape[1]
pvals_adj[pvals_adj > 1] = 1 # cap adjusted p-value at 1 pvals_adj[pvals_adj > 1] = 1 # cap adjusted p-value at 1
# logfoldchanges: log2(meanA / meanB) # logfoldchanges: log2(meanA / meanB)
logfoldchanges = np.log2(np.abs((meanA + 1e-9) / (meanB + 1e-9))) logfoldchanges = np.log2(np.abs((meanA + 1e-9) / (meanB + 1e-9)))
@@ -106,8 +105,5 @@ def diffexp_ttest(adata, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
pvals_adj_top_n = pvals_adj[sort_order] pvals_adj_top_n = pvals_adj[sort_order]
# varIndex, logfoldchange, pval, pval_adj # varIndex, logfoldchange, pval, pval_adj
result = [[sort_order[i], result = [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n)]
logfoldchanges_top_n[i],
pvals_top_n[i],
pvals_adj_top_n[i]] for i in range(top_n)]
return result return result
server/app/scanpy_engine/scanpy_engine.py
+53 -62
View File
@@ -22,7 +22,6 @@ Sort order for methods
class ScanpyEngine(CXGDriver): class ScanpyEngine(CXGDriver):
def __init__(self, data, args): def __init__(self, data, args):
super().__init__(data, args) super().__init__(data, args)
self._alias_annotation_names(Axis.OBS, args["obs_names"]) self._alias_annotation_names(Axis.OBS, args["obs_names"])
@@ -36,7 +35,7 @@ class ScanpyEngine(CXGDriver):
self._create_schema() self._create_schema()
# TODO: temporary work-arounds # TODO: temporary work-arounds
if args['nan_to_num']: if args["nan_to_num"]:
self._IEEE754_special_values_workaround() self._IEEE754_special_values_workaround()
def _alias_annotation_names(self, axis, name): def _alias_annotation_names(self, axis, name):
@@ -61,8 +60,9 @@ class ScanpyEngine(CXGDriver):
if name not in df_axis.columns: if name not in df_axis.columns:
raise KeyError(f"Annotation name {name}, specified in --{ax_name}-name does not exist.") raise KeyError(f"Annotation name {name}, specified in --{ax_name}-name does not exist.")
if not df_axis[name].is_unique: if not df_axis[name].is_unique:
raise KeyError(f"Values in -{ax_name}-name must be unique. " raise KeyError(
"Please prepare data to contain unique values.") f"Values in -{ax_name}-name must be unique. " "Please prepare data to contain unique values."
)
# reset index to simple range; alias user-specified annotation to "name" # reset index to simple range; alias user-specified annotation to "name"
df_axis.reset_index(drop=True, inplace=True) df_axis.reset_index(drop=True, inplace=True)
df_axis.rename(inplace=True, columns={name: "name"}) df_axis.rename(inplace=True, columns={name: "name"})
@@ -89,15 +89,8 @@ class ScanpyEngine(CXGDriver):
def _create_schema(self): def _create_schema(self):
self.schema = { self.schema = {
"dataframe": { "dataframe": {"nObs": self.cell_count, "nVar": self.gene_count, "type": str(self.data.X.dtype)},
"nObs": self.cell_count, "annotations": {"obs": [], "var": []},
"nVar": self.gene_count,
"type": str(self.data.X.dtype)
},
"annotations": {
"obs": [],
"var": []
}
} }
for ax in Axis: for ax in Axis:
curr_axis = getattr(self.data, str(ax)) curr_axis = getattr(self.data, str(ax))
@@ -129,32 +122,35 @@ class ScanpyEngine(CXGDriver):
try: try:
result = sc.read(data, cache=True) result = sc.read(data, cache=True)
except ValueError: except ValueError:
raise ScanpyFileError("File must be in the .h5ad format. Please read " raise ScanpyFileError(
"https://github.com/theislab/scanpy_usage/blob/master/170505_seurat/info_h5ad.md to " "File must be in the .h5ad format. Please read "
"learn more about this format. You may be able to convert your file into this format " "https://github.com/theislab/scanpy_usage/blob/master/170505_seurat/info_h5ad.md to "
"using `cellxgene prepare`, please run `cellxgene prepare --help` for more " "learn more about this format. You may be able to convert your file into this format "
"information.") "using `cellxgene prepare`, please run `cellxgene prepare --help` for more "
"information."
)
except Exception as e: except Exception as e:
raise ScanpyFileError(f"Error while loading file: {e}, File must be in the .h5ad format, please check " raise ScanpyFileError(
f"that your input and try again.") f"Error while loading file: {e}, File must be in the .h5ad format, please check "
f"that your input and try again."
)
return result return result
def _validate_data_types(self): def _validate_data_types(self):
if self.data.X.dtype != "float32": if self.data.X.dtype != "float32":
warnings.warn(f"Scanpy data matrix is in {self.data.X.dtype} format not float32. " warnings.warn(
f"Precision may be truncated.") f"Scanpy data matrix is in {self.data.X.dtype} format not float32. " f"Precision may be truncated."
)
for ax in Axis: for ax in Axis:
curr_axis = getattr(self.data, str(ax)) curr_axis = getattr(self.data, str(ax))
for ann in curr_axis: for ann in curr_axis:
datatype = curr_axis[ann].dtype datatype = curr_axis[ann].dtype
downcast_map = {"int64": "int32", downcast_map = {"int64": "int32", "uint32": "int32", "uint64": "int32", "float64": "float32"}
"uint32": "int32",
"uint64": "int32",
"float64": "float32",
}
if datatype in downcast_map: if datatype in downcast_map:
warnings.warn(f"Scanpy annotation {ax}:{ann} is in unsupported format: {datatype}. " warnings.warn(
f"Data will be downcast to {downcast_map[datatype]}.") f"Scanpy annotation {ax}:{ann} is in unsupported format: {datatype}. "
f"Data will be downcast to {downcast_map[datatype]}."
)
if isinstance(datatype, CategoricalDtype): if isinstance(datatype, CategoricalDtype):
category_num = len(curr_axis[ann].dtype.categories) category_num = len(curr_axis[ann].dtype.categories)
if category_num > 500 and category_num > self.max_category_items: if category_num > 500 and category_num > self.max_category_items:
@@ -162,7 +158,8 @@ class ScanpyEngine(CXGDriver):
f"{str(ax).title()} annotation '{ann}' has {category_num} categories, this may be " f"{str(ax).title()} annotation '{ann}' has {category_num} categories, this may be "
f"cumbersome or slow to display. We recommend setting the " f"cumbersome or slow to display. We recommend setting the "
f"--max-category-items option to 500, this will hide categorical " f"--max-category-items option to 500, this will hide categorical "
f"annotations with more than 500 categories in the UI") f"annotations with more than 500 categories in the UI"
)
def _validate_data_calculations(self): def _validate_data_calculations(self):
layout_key = f"X_{self.layout_method}" layout_key = f"X_{self.layout_method}"
@@ -174,7 +171,8 @@ class ScanpyEngine(CXGDriver):
f" layout may have been computed. The requested layout must be pre-calculated and saved " f" layout may have been computed. The requested layout must be pre-calculated and saved "
f"back in the h5ad file. You can run " f"back in the h5ad file. You can run "
f"`cellxgene prepare --layout {self.layout_method} <datafile>` " f"`cellxgene prepare --layout {self.layout_method} <datafile>` "
f"to solve this problem. ") f"to solve this problem. "
)
def _IEEE754_special_values_workaround(self): def _IEEE754_special_values_workaround(self):
""" """
@@ -196,7 +194,7 @@ class ScanpyEngine(CXGDriver):
curr_axis = getattr(self.data, str(ax)) curr_axis = getattr(self.data, str(ax))
for ann in curr_axis: for ann in curr_axis:
dtype = curr_axis[ann].dtype dtype = curr_axis[ann].dtype
if dtype.kind == 'f': if dtype.kind == "f":
finite_idx = np.isfinite(curr_axis[ann]) finite_idx = np.isfinite(curr_axis[ann])
if not finite_idx.all(): if not finite_idx.all():
curr_axis.loc[np.isnan(curr_axis[ann]), ann] = 0 curr_axis.loc[np.isnan(curr_axis[ann]), ann] = 0
@@ -233,8 +231,7 @@ class ScanpyEngine(CXGDriver):
if non_finite_X_found: if non_finite_X_found:
warnings.warn( warnings.warn(
"Dataframe X contains floating point NaN or Infinities. " "Dataframe X contains floating point NaN or Infinities. " "These will be converted to finite values."
"These will be converted to finite values."
) )
def filter_dataframe(self, filter): def filter_dataframe(self, filter):
@@ -256,7 +253,7 @@ class ScanpyEngine(CXGDriver):
@staticmethod @staticmethod
def _annotation_filter_to_mask(filter, d_axis, count): def _annotation_filter_to_mask(filter, d_axis, count):
mask = np.ones((count, ), dtype=bool) mask = np.ones((count,), dtype=bool)
for v in filter: for v in filter:
if d_axis[v["name"]].dtype.name in ["boolean", "category", "object"]: if d_axis[v["name"]].dtype.name in ["boolean", "category", "object"]:
key_idx = np.in1d(getattr(d_axis, v["name"]), v["values"]) key_idx = np.in1d(getattr(d_axis, v["name"]), v["values"])
@@ -274,24 +271,23 @@ class ScanpyEngine(CXGDriver):
@staticmethod @staticmethod
def _index_filter_to_mask(filter, count): def _index_filter_to_mask(filter, count):
mask = np.zeros((count, ), dtype=bool) mask = np.zeros((count,), dtype=bool)
for i in filter: for i in filter:
if type(i) == list: if type(i) == list:
mask[i[0]:i[1]] = True mask[i[0] : i[1]] = True
else: else:
mask[i] = True mask[i] = True
return mask return mask
@staticmethod @staticmethod
def _axis_filter_to_mask(filter, d_axis, count): def _axis_filter_to_mask(filter, d_axis, count):
mask = np.ones((count, ), dtype=bool) mask = np.ones((count,), dtype=bool)
if "index" in filter: if "index" in filter:
mask = np.logical_and(mask, ScanpyEngine._index_filter_to_mask(filter["index"], count)) mask = np.logical_and(mask, ScanpyEngine._index_filter_to_mask(filter["index"], count))
if "annotation_value" in filter: if "annotation_value" in filter:
mask = np.logical_and(mask, mask = np.logical_and(
ScanpyEngine._annotation_filter_to_mask(filter["annotation_value"], mask, ScanpyEngine._annotation_filter_to_mask(filter["annotation_value"], d_axis, count)
d_axis, )
count))
return mask return mask
def _filter_to_mask(self, filter, use_slices=True): def _filter_to_mask(self, filter, use_slices=True):
@@ -321,8 +317,9 @@ class ScanpyEngine(CXGDriver):
https://docs.scipy.org/doc/scipy/reference/sparse.html https://docs.scipy.org/doc/scipy/reference/sparse.html
""" """
prefer_row_access = sparse.isspmatrix_csr(data._X) or sparse.isspmatrix_lil(data._X) \ prefer_row_access = (
or sparse.isspmatrix_bsr(data._X) sparse.isspmatrix_csr(data._X) or sparse.isspmatrix_lil(data._X) or sparse.isspmatrix_bsr(data._X)
)
if prefer_row_access: if prefer_row_access:
# Row-major slicing # Row-major slicing
if obs_selector is not None: if obs_selector is not None:
@@ -355,18 +352,12 @@ class ScanpyEngine(CXGDriver):
obs = self.data.obs[obs_selector] obs = self.data.obs[obs_selector]
if not fields: if not fields:
fields = obs.columns.tolist() fields = obs.columns.tolist()
result = { result = {"names": fields, "data": DataFrame(obs[fields]).to_records(index=True).tolist()}
"names": fields,
"data": DataFrame(obs[fields]).to_records(index=True).tolist()
}
else: else:
var = self.data.var[var_selector] var = self.data.var[var_selector]
if not fields: if not fields:
fields = var.columns.tolist() fields = var.columns.tolist()
result = { result = {"names": fields, "data": DataFrame(var[fields]).to_records(index=True).tolist()}
"names": fields,
"data": DataFrame(var[fields]).to_records(index=True).tolist()
}
return result return result
def data_frame(self, filter, axis): def data_frame(self, filter, axis):
@@ -391,12 +382,12 @@ class ScanpyEngine(CXGDriver):
if axis == Axis.OBS: if axis == Axis.OBS:
result = { result = {
"var": var_index_sliced.tolist(), "var": var_index_sliced.tolist(),
"obs": DataFrame(_X, index=obs_index_sliced).to_records(index=True).tolist() "obs": DataFrame(_X, index=obs_index_sliced).to_records(index=True).tolist(),
} }
else: else:
result = { result = {
"obs": obs_index_sliced.tolist(), "obs": obs_index_sliced.tolist(),
"var": DataFrame(_X.T, index=var_index_sliced).to_records(index=True).tolist() "var": DataFrame(_X.T, index=var_index_sliced).to_records(index=True).tolist(),
} }
return result return result
@@ -435,11 +426,11 @@ class ScanpyEngine(CXGDriver):
try: try:
df_layout = df.obsm[f"X_{self.layout_method}"] df_layout = df.obsm[f"X_{self.layout_method}"]
except ValueError as e: except ValueError as e:
raise PrepareError(f"Layout has not been calculated using {self.layout_method}, " raise PrepareError(
f"please prepare your datafile and relaunch cellxgene") from e f"Layout has not been calculated using {self.layout_method}, "
normalized_layout = DataFrame((df_layout - df_layout.min()) / (df_layout.max() - df_layout.min()), f"please prepare your datafile and relaunch cellxgene"
index=df.obs.index) ) from e
return { normalized_layout = DataFrame(
"ndims": normalized_layout.shape[1], (df_layout - df_layout.min()) / (df_layout.max() - df_layout.min()), index=df.obs.index
"coordinates": normalized_layout.to_records(index=True).tolist() )
} return {"ndims": normalized_layout.shape[1], "coordinates": normalized_layout.to_records(index=True).tolist()}
server/app/util/filter.py
-1
View File
@@ -7,7 +7,6 @@ from server.app.util.constants import Axis
class QueryStringError(Exception): class QueryStringError(Exception):
def __init__(self, key, message): def __init__(self, key, message):
self.key = key self.key = key
self.message = message self.message = message
server/app/util/models.py
+7 -38
View File
@@ -5,24 +5,13 @@ class AnnotationModel(Schema):
type = "object" type = "object"
description = "Filter by annotation key: value" description = "Filter by annotation key: value"
properties = { properties = {
"name": { "name": {"type": "string"},
"type": "string"
},
# TODO update to OpenAPI v3.0 when a library is available that supports it # TODO update to OpenAPI v3.0 when a library is available that supports it
# Unfortunately 2.0 doesn't have a way to have a schema that accepts multiple types # Unfortunately 2.0 doesn't have a way to have a schema that accepts multiple types
# Overloading the type key with a list seems to work ok and makes it to the page # Overloading the type key with a list seems to work ok and makes it to the page
"values": { "values": {"type": "array", "items": {"type": ["float32", "string", "int32", "bool"]}},
"type": "array", "min": {"type": ["int32", "float32"]},
"items": { "max": {"type": ["int32", "float32"]},
"type": ["float32", "string", "int32", "bool"]
}
},
"min": {
"type": ["int32", "float32"],
},
"max": {
"type": ["int32", "float32"],
}
} }
required = ["name"] required = ["name"]
@@ -30,36 +19,16 @@ class AnnotationModel(Schema):
class IndexModel(Schema): class IndexModel(Schema):
type = "object" type = "object"
description = "Filter by index of observation/variable ex. [0, 5, 15]" description = "Filter by index of observation/variable ex. [0, 5, 15]"
properties = { properties = {"index": {"type": "array", "items": {"format": "int32", "type": "integer"}}}
"index": {
"type": "array",
"items": {
"format": "int32",
"type": "integer"
}
}
}
class AxisModel(Schema): class AxisModel(Schema):
type = "object" type = "object"
description = "Axis of data -- obs or var" description = "Axis of data -- obs or var"
properties = { properties = {"index": IndexModel, "annotation_value": AnnotationModel.array()}
"index": IndexModel,
"annotation_value": AnnotationModel.array()
}
class FilterModel(Schema): class FilterModel(Schema):
type = "object" type = "object"
description = "Complex filter" description = "Complex filter"
properties = { properties = {"filter": {"type": "object", "properties": {"obs": AxisModel, "var": AxisModel}}}
"filter": {
"type": "object",
"properties": {
"obs": AxisModel,
"var": AxisModel
}
}
}
server/app/util/utils.py
+4 -3
View File
@@ -15,7 +15,7 @@ class Float32JSONEncoder(json.JSONEncoder):
if it runs into non-finite floating point values which are unsupported by if it runs into non-finite floating point values which are unsupported by
standard JSON. standard JSON.
""" """
kwargs['allow_nan'] = False kwargs["allow_nan"] = False
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
def default(self, obj): def default(self, obj):
@@ -30,8 +30,9 @@ def custom_format_warning(msg, *args, **kwargs):
return f"[cellxgene] Warning: {msg} \n" return f"[cellxgene] Warning: {msg} \n"
def get_mime_type(default="application/json", acceptable_types=["application/json", "text/csv"], query_param=None, def get_mime_type(
header=None): default="application/json", acceptable_types=["application/json", "text/csv"], query_param=None, header=None
):
mime_type = default mime_type = default
if query_param: if query_param:
if query_param in acceptable_types: if query_param in acceptable_types:
server/app/web/webapp.py
+1 -3
View File
@@ -1,7 +1,5 @@
import os import os
from flask import ( from flask import Blueprint, render_template, send_from_directory, current_app
Blueprint, render_template, send_from_directory, current_app
)
bp = Blueprint("webapp", __name__, template_folder="templates") bp = Blueprint("webapp", __name__, template_folder="templates")
server/cli/launch.py
+70 -26
View File
@@ -13,30 +13,76 @@ from server.app.util.utils import custom_format_warning
@click.command() @click.command()
@click.argument("data", metavar="<data file>", type=click.Path(exists=True, file_okay=True, dir_okay=False)) @click.argument("data", metavar="<data file>", type=click.Path(exists=True, file_okay=True, dir_okay=False))
@click.option("--layout", "-l", type=click.Choice(["umap", "tsne"]), default="umap", show_default=True, @click.option(
help="Method for layout.") "--layout", "-l", type=click.Choice(["umap", "tsne"]), default="umap", show_default=True, help="Method for layout."
@click.option("--diffexp", "-d", type=click.Choice(["ttest"]), default="ttest", show_default=True, )
help="Method for differential expression.") @click.option(
"--diffexp",
"-d",
type=click.Choice(["ttest"]),
default="ttest",
show_default=True,
help="Method for differential expression.",
)
@click.option("--title", "-t", help="Title to display (if omitted will use file name).", metavar="") @click.option("--title", "-t", help="Title to display (if omitted will use file name).", metavar="")
@click.option("--verbose", "-v", is_flag=True, default=False, show_default=True, @click.option(
help="Provide verbose output, including warnings and all server requests.") "--verbose",
@click.option("--debug", "-d", is_flag=True, default=False, show_default=True, "-v",
help="Run in debug mode.") is_flag=True,
@click.option("--open", "-o", "open_browser", is_flag=True, default=False, show_default=True, default=False,
help="Open the web browser after launch.") show_default=True,
help="Provide verbose output, including warnings and all server requests.",
)
@click.option("--debug", "-d", is_flag=True, default=False, show_default=True, help="Run in debug mode.")
@click.option(
"--open",
"-o",
"open_browser",
is_flag=True,
default=False,
show_default=True,
help="Open the web browser after launch.",
)
@click.option("--port", "-p", help="Port to run server on.", metavar="", default=5005, show_default=True) @click.option("--port", "-p", help="Port to run server on.", metavar="", default=5005, show_default=True)
@click.option("--obs-names", default=None, metavar="", help="Name of annotation field to use for observations.") @click.option("--obs-names", default=None, metavar="", help="Name of annotation field to use for observations.")
@click.option("--var-names", default=None, metavar="", help="Name of annotation to use for variables.") @click.option("--var-names", default=None, metavar="", help="Name of annotation to use for variables.")
@click.option("--host", default="127.0.0.1", help="Host IP address") @click.option("--host", default="127.0.0.1", help="Host IP address")
@click.option("--max-category-items", default=100, metavar="", show_default=True, @click.option(
help="Limits the number of categorical annotation items displayed.") "--max-category-items",
@click.option("--diffexp-lfc-cutoff", default=0.01, show_default=True, default=100,
help="Relative expression cutoff used when selecting top N differentially expressed genes") metavar="",
@click.option("--nan-to-num", is_flag=True, default=False, show_default=True, show_default=True,
help="Replace all floating point NaN with zero, and infinities with finite numbers") help="Limits the number of categorical annotation items displayed.",
def launch(data, layout, diffexp, title, verbose, debug, obs_names, var_names, )
open_browser, port, host, max_category_items, diffexp_lfc_cutoff, @click.option(
nan_to_num): "--diffexp-lfc-cutoff",
default=0.01,
show_default=True,
help="Relative expression cutoff used when selecting top N differentially expressed genes",
)
@click.option(
"--nan-to-num",
is_flag=True,
default=False,
show_default=True,
help="Replace all floating point NaN with zero, and infinities with finite numbers",
)
def launch(
data,
layout,
diffexp,
title,
verbose,
debug,
obs_names,
var_names,
open_browser,
port,
host,
max_category_items,
diffexp_lfc_cutoff,
nan_to_num,
):
"""Launch the cellxgene data viewer. """Launch the cellxgene data viewer.
This web app lets you explore single-cell expression data. This web app lets you explore single-cell expression data.
Data must be in a format that cellxgene expects, read the Data must be in a format that cellxgene expects, read the
@@ -76,10 +122,7 @@ def launch(data, layout, diffexp, title, verbose, debug, obs_names, var_names,
# Import Flask app # Import Flask app
from server.app.app import app from server.app.app import app
app.config.update( app.config.update(DATASET_TITLE=title, CXG_API_BASE=api_base)
DATASET_TITLE=title,
CXG_API_BASE=api_base
)
if not verbose: if not verbose:
log = logging.getLogger("werkzeug") log = logging.getLogger("werkzeug")
@@ -90,7 +133,8 @@ def launch(data, layout, diffexp, title, verbose, debug, obs_names, var_names,
# Fix for anaconda python. matplotlib typically expects python to be installed as a framework TKAgg is usually # Fix for anaconda python. matplotlib typically expects python to be installed as a framework TKAgg is usually
# available and fixes this issue. See https://matplotlib.org/faq/virtualenv_faq.html # available and fixes this issue. See https://matplotlib.org/faq/virtualenv_faq.html
import matplotlib as mpl import matplotlib as mpl
mpl.use('TkAgg')
mpl.use("TkAgg")
from server.app.scanpy_engine.scanpy_engine import ScanpyEngine from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
args = { args = {
@@ -100,7 +144,7 @@ def launch(data, layout, diffexp, title, verbose, debug, obs_names, var_names,
"diffexp_lfc_cutoff": diffexp_lfc_cutoff, "diffexp_lfc_cutoff": diffexp_lfc_cutoff,
"obs_names": obs_names, "obs_names": obs_names,
"var_names": var_names, "var_names": var_names,
"nan_to_num": nan_to_num "nan_to_num": nan_to_num,
} }
try: try:
@@ -117,7 +161,7 @@ def launch(data, layout, diffexp, title, verbose, debug, obs_names, var_names,
click.echo("[cellxgene] Type CTRL-C at any time to exit.") click.echo("[cellxgene] Type CTRL-C at any time to exit.")
if not verbose: if not verbose:
f = open(devnull, 'w') f = open(devnull, "w")
sys.stdout = f sys.stdout = f
app.run(host=host, debug=debug, port=port, threaded=True) app.run(host=host, debug=debug, port=port, threaded=True)
server/cli/prepare.py
+48 -17
View File
@@ -7,22 +7,48 @@ from scipy.sparse.csc import csc_matrix
@click.command() @click.command()
@click.argument("data", nargs=1, metavar="<dataset: file or path to data>", required=True) @click.argument("data", nargs=1, metavar="<dataset: file or path to data>", required=True)
@click.option("--layout", "-l", default=["umap", "tsne"], multiple=True, type=click.Choice(["umap", "tsne"]), @click.option(
help="Layout algorithm", show_default=True) "--layout",
@click.option("--recipe", "-r", default="none", type=click.Choice(["none", "seurat", "zheng17"]), "-l",
help="Preprocessing to run.", show_default=True) default=["umap", "tsne"],
multiple=True,
type=click.Choice(["umap", "tsne"]),
help="Layout algorithm",
show_default=True,
)
@click.option(
"--recipe",
"-r",
default="none",
type=click.Choice(["none", "seurat", "zheng17"]),
help="Preprocessing to run.",
show_default=True,
)
@click.option("--output", "-o", default="", help="Save a new file to filename.", metavar="<filename>") @click.option("--output", "-o", default="", help="Save a new file to filename.", metavar="<filename>")
@click.option("--plotting", "-p", default=False, is_flag=True, help="Whether to generate plots.", show_default=True) @click.option("--plotting", "-p", default=False, is_flag=True, help="Whether to generate plots.", show_default=True)
@click.option("--sparse", default=False, is_flag=True, help="Whether to force sparsity.", show_default=True) @click.option("--sparse", default=False, is_flag=True, help="Whether to force sparsity.", show_default=True)
@click.option("--overwrite", default=False, is_flag=True, help="Allow file overwriting.", show_default=True) @click.option("--overwrite", default=False, is_flag=True, help="Allow file overwriting.", show_default=True)
@click.option("--set-obs-names", default="", help="Named field to set as index for obs.", metavar="<name>") @click.option("--set-obs-names", default="", help="Named field to set as index for obs.", metavar="<name>")
@click.option("--set-var-names", default="", help="Named field to set as index for var.", metavar="<name>") @click.option("--set-var-names", default="", help="Named field to set as index for var.", metavar="<name>")
@click.option("--make-obs-names-unique", default=True, is_flag=True, @click.option(
help="Ensure obs index is unique.", show_default=True) "--make-obs-names-unique", default=True, is_flag=True, help="Ensure obs index is unique.", show_default=True
@click.option("--make-var-names-unique", default=True, is_flag=True, )
help="Ensure var index is unique.", show_default=True) @click.option(
def prepare(data, layout, recipe, output, plotting, sparse, overwrite, "--make-var-names-unique", default=True, is_flag=True, help="Ensure var index is unique.", show_default=True
set_obs_names, set_var_names, make_obs_names_unique, make_var_names_unique): )
def prepare(
data,
layout,
recipe,
output,
plotting,
sparse,
overwrite,
set_obs_names,
set_var_names,
make_obs_names_unique,
make_var_names_unique,
):
"""Preprocesses data for use with cellxgene. """Preprocesses data for use with cellxgene.
This tool runs a series of scanpy routines for preparing a dataset This tool runs a series of scanpy routines for preparing a dataset
@@ -35,6 +61,7 @@ def prepare(data, layout, recipe, output, plotting, sparse, overwrite,
# collect slow imports here to make CLI startup more responsive # collect slow imports here to make CLI startup more responsive
click.echo("[cellxgene] Starting CLI...") click.echo("[cellxgene] Starting CLI...")
import matplotlib import matplotlib
matplotlib.use("Agg") matplotlib.use("Agg")
import scanpy.api as sc import scanpy.api as sc
@@ -49,8 +76,10 @@ def prepare(data, layout, recipe, output, plotting, sparse, overwrite,
output = expanduser(output) output = expanduser(output)
if not output: if not output:
click.echo("Warning: No file will be saved, to save the results of cellxgene prepare include " click.echo(
"--output <filename> to save output to a new file") "Warning: No file will be saved, to save the results of cellxgene prepare include "
"--output <filename> to save output to a new file"
)
if isfile(output) and not overwrite: if isfile(output) and not overwrite:
raise click.UsageError(f"Cannot overwrite existing file {output}, try using the flag --overwrite") raise click.UsageError(f"Cannot overwrite existing file {output}, try using the flag --overwrite")
@@ -119,9 +148,11 @@ def prepare(data, layout, recipe, output, plotting, sparse, overwrite,
try: try:
sc.tl.louvain(adata) sc.tl.louvain(adata)
except ModuleNotFoundError: except ModuleNotFoundError:
click.echo("\nWarning: louvain module is not installed, no clusters will be calculated. " click.echo(
"To fix this please install cellxgene with the optional feature louvain enabled: " "\nWarning: louvain module is not installed, no clusters will be calculated. "
"`pip install cellxgene[louvain]`") "To fix this please install cellxgene with the optional feature louvain enabled: "
"`pip install cellxgene[louvain]`"
)
def run_layout(adata): def run_layout(adata):
if len(unique(adata.obs["louvain"].values)) < 10: if len(unique(adata.obs["louvain"].values)) < 10:
@@ -142,11 +173,11 @@ def prepare(data, layout, recipe, output, plotting, sparse, overwrite,
def show_step(item): def show_step(item):
names = { names = {
"make_sparse": "Ensuring sparsity", "make_sparse": "Ensuring sparsity",
"run_recipe": f"Running preprocessing recipe \"{recipe}\"", "run_recipe": f'Running preprocessing recipe "{recipe}"',
"run_pca": "Running PCA", "run_pca": "Running PCA",
"run_neighbors": "Calculating neighbors", "run_neighbors": "Calculating neighbors",
"run_louvain": "Calculating clusters", "run_louvain": "Calculating clusters",
"run_layout": "Computing layout" "run_layout": "Computing layout",
} }
if item is not None: if item is not None:
return names[item.__name__] return names[item.__name__]
server/requirements-dev.txt
+2 -1
View File
@@ -1,5 +1,6 @@
black
bumpversion>=0.5
pytest>=3.6.3 pytest>=3.6.3
requests>=2.18.4 requests>=2.18.4
twine>=1.12.1 twine>=1.12.1
bumpversion>=0.5
-r requirements.txt -r requirements.txt
server/test/test_api.py
+59 -80
View File
@@ -9,15 +9,7 @@ LOCAL_URL = "http://127.0.0.1:5005/"
VERSION = "v0.2" VERSION = "v0.2"
URL_BASE = f"{LOCAL_URL}api/{VERSION}/" URL_BASE = f"{LOCAL_URL}api/{VERSION}/"
BAD_FILTER = { BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
"filter": {
"obs": {
"annotation_value": [
{"name": "xyz"},
],
}
}
}
class EndPoints(unittest.TestCase): class EndPoints(unittest.TestCase):
@@ -133,7 +125,7 @@ class EndPoints(unittest.TestCase):
{"name": "louvain", "values": ["NK cells", "CD8 T cells"]}, {"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
{"name": "n_counts", "min": 3000}, {"name": "n_counts", "min": 3000},
], ],
"index": [1, 99, [1000, 2000]] "index": [1, 99, [1000, 2000]],
} }
} }
} }
@@ -154,7 +146,7 @@ class EndPoints(unittest.TestCase):
{"name": "louvain", "values": ["NK cells", "CD8 T cells"]}, {"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
{"name": "n_counts", "min": 3000}, {"name": "n_counts", "min": 3000},
], ],
"index": [1, 99, [1000, 2000]] "index": [1, 99, [1000, 2000]],
} }
} }
} }
@@ -170,23 +162,9 @@ class EndPoints(unittest.TestCase):
url = f"{URL_BASE}{endpoint}" url = f"{URL_BASE}{endpoint}"
params = { params = {
"mode": "topN", "mode": "topN",
"set1": { "set1": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["NK cells"]}]}}},
"filter": { "set2": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["CD8 T cells"]}]}}},
"obs": {"annotation_value": [ "count": 7,
{"name": "louvain", "values": ["NK cells"]}
]
}
}
},
"set2": {
"filter": {
"obs": {"annotation_value": [
{"name": "louvain", "values": ["CD8 T cells"]}
]
}
}
},
"count": 7
} }
result = self.session.post(url, json=params) result = self.session.post(url, json=params)
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
@@ -199,20 +177,8 @@ class EndPoints(unittest.TestCase):
params = { params = {
"mode": "topN", "mode": "topN",
"count": 10, "count": 10,
"set1": { "set1": {"filter": {"obs": {"index": [[0, 500]]}}},
"filter": { "set2": {"filter": {"obs": {"index": [[500, 1000]]}}},
"obs": {
"index": [[0, 500]]
}
}
},
"set2": {
"filter": {
"obs": {
"index": [[500, 1000]]
}
}
}
} }
result = self.session.post(url, json=params) result = self.session.post(url, json=params)
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
@@ -249,15 +215,7 @@ class EndPoints(unittest.TestCase):
def test_put_annotations_var(self): def test_put_annotations_var(self):
endpoint = "annotations/var" endpoint = "annotations/var"
url = f"{URL_BASE}{endpoint}" url = f"{URL_BASE}{endpoint}"
var_filter = { var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}]}}}
"filter": {
"var": {
"annotation_value": [
{"name": "name", "values": ["ATAD3C", "RER1"]},
]
}
}
}
result = self.session.put(url, json=var_filter) result = self.session.put(url, json=var_filter)
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
result_data = result.json() result_data = result.json()
@@ -268,15 +226,7 @@ class EndPoints(unittest.TestCase):
endpoint = "annotations/var" endpoint = "annotations/var"
query = "annotation-name=n_cells" query = "annotation-name=n_cells"
url = f"{URL_BASE}{endpoint}?{query}" url = f"{URL_BASE}{endpoint}?{query}"
var_filter = { var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}]}}}
"filter": {
"var": {
"annotation_value": [
{"name": "name", "values": ["ATAD3C", "RER1"]},
]
}
}
}
result = self.session.put(url, json=var_filter) result = self.session.put(url, json=var_filter)
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
result_data = result.json() result_data = result.json()
@@ -335,7 +285,7 @@ class EndPoints(unittest.TestCase):
{"name": "louvain", "values": ["NK cells", "CD8 T cells"]}, {"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
{"name": "n_counts", "min": 3000}, {"name": "n_counts", "min": 3000},
], ],
"index": [1, 99, [1000, 2000]] "index": [1, 99, [1000, 2000]],
} }
} }
} }
@@ -349,15 +299,7 @@ class EndPoints(unittest.TestCase):
endpoint = f"data/{axis}" endpoint = f"data/{axis}"
url = f"{URL_BASE}{endpoint}" url = f"{URL_BASE}{endpoint}"
header = {"Accept": "application/json"} header = {"Accept": "application/json"}
var_filter = { var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}}}
"filter": {
"var": {
"annotation_value": [
{"name": "name", "values": ["RER1"]},
]
}
}
}
result = self.session.put(url, headers=header, json=var_filter) result = self.session.put(url, headers=header, json=var_filter)
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
result_data = result.json() result_data = result.json()
@@ -371,16 +313,44 @@ class EndPoints(unittest.TestCase):
def test_cache(self): def test_cache(self):
endpoint = "annotations/var" endpoint = "annotations/var"
url = f"{URL_BASE}{endpoint}" url = f"{URL_BASE}{endpoint}"
f1 = {"filter": {"var": {"annotation_value": [{"name": "name", f1 = {
"values": ["HLA-DRB1", "HLA-DQA1", "HLA-DQB1", "HLA-DPA1", "filter": {
"HLA-DPB1", "MS4A1", "IL32", "CCL5", "CD79B", "var": {
"CD79A"]}]}}} "annotation_value": [
{
"name": "name",
"values": [
"HLA-DRB1",
"HLA-DQA1",
"HLA-DQB1",
"HLA-DPA1",
"HLA-DPB1",
"MS4A1",
"IL32",
"CCL5",
"CD79B",
"CD79A",
],
}
]
}
}
}
result = self.session.put(url, json=f1) result = self.session.put(url, json=f1)
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
result_data1 = result.json() result_data1 = result.json()
f2 = {"filter": {"var": {"annotation_value": [{"name": "name", f2 = {
"values": ["FGFBP2", "GZMA", "LTB", "PRF1", "CTSW", "GZMH", "filter": {
"CCL5", "CCL4", "CST7", "NKG7"]}]}}} "var": {
"annotation_value": [
{
"name": "name",
"values": ["FGFBP2", "GZMA", "LTB", "PRF1", "CTSW", "GZMH", "CCL5", "CCL4", "CST7", "NKG7"],
}
]
}
}
}
result = self.session.put(url, json=f2) result = self.session.put(url, json=f2)
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
result_data2 = result.json() result_data2 = result.json()
@@ -393,9 +363,18 @@ class EndPoints(unittest.TestCase):
result = self.session.put(url, json=f1) result = self.session.put(url, json=f1)
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
result_data1 = result.json() result_data1 = result.json()
f2 = {"filter": {"var": {"annotation_value": [{"name": "name", f2 = {
"values": ["FGFBP2", "GZMA", "LTB", "PRF1", "CTSW", "GZMH", "filter": {
"CCL5", "CCL4", "CST7", "NKG7"]}]}}} "var": {
"annotation_value": [
{
"name": "name",
"values": ["FGFBP2", "GZMA", "LTB", "PRF1", "CTSW", "GZMH", "CCL5", "CCL4", "CST7", "NKG7"],
}
]
}
}
}
result = self.session.put(url, json=f2) result = self.session.put(url, json=f2)
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
result_data2 = result.json() result_data2 = result.json()
server/test/test_filter.py
+10 -8
View File
@@ -54,13 +54,17 @@ class UtilTest(unittest.TestCase):
def test_complex_filter(self): def test_complex_filter(self):
filter_dict = ImmutableMultiDict( filter_dict = ImmutableMultiDict(
[("obs:louvain", "NK cells"), ("obs:louvain", "CD8 T cells"), ("obs:n_counts", "3000,*")]) [("obs:louvain", "NK cells"), ("obs:louvain", "CD8 T cells"), ("obs:n_counts", "3000,*")]
)
filter_ = parse_filter(filter_dict, self.schema) filter_ = parse_filter(filter_dict, self.schema)
self.assertIn("obs", filter_) self.assertIn("obs", filter_)
self.assertEqual(filter_["obs"]["annotation_value"], [{"name": "louvain", self.assertEqual(
"values": ["NK cells", "CD8 T cells"]}, filter_["obs"]["annotation_value"],
{"name": "n_counts", [
"max": None, "min": 3000.0}]) {"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
{"name": "n_counts", "max": None, "min": 3000.0},
],
)
def test_bad_filter(self): def test_bad_filter(self):
bad_annotation_type = ImmutableMultiDict([("obs:tissue", "lung")]) bad_annotation_type = ImmutableMultiDict([("obs:tissue", "lung")])
@@ -71,9 +75,7 @@ class UtilTest(unittest.TestCase):
parse_filter(bad_axis, self.schema) parse_filter(bad_axis, self.schema)
def test_boolean_filter(self): def test_boolean_filter(self):
schema = { schema = {"obs": [{"name": "bool_filter", "type": "boolean"}]}
"obs": [{"name": "bool_filter", "type": "boolean"}]
}
filter_dict = ImmutableMultiDict([("obs:bool_filter", "false")]) filter_dict = ImmutableMultiDict([("obs:bool_filter", "false")])
filter_ = parse_filter(filter_dict, schema) filter_ = parse_filter(filter_dict, schema)
self.assertIn("obs", filter_) self.assertIn("obs", filter_)
server/test/test_scanpy_engine.py
+30 -101
View File
@@ -3,7 +3,6 @@ from os import path
import pytest import pytest
import time import time
import unittest import unittest
import argparse
import numpy as np import numpy as np
from pandas import Series from pandas import Series
@@ -13,9 +12,15 @@ from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
class UtilTest(unittest.TestCase): class UtilTest(unittest.TestCase):
def setUp(self): def setUp(self):
args = {'layout': 'umap', 'diffexp': 'ttest', 'max_category_items': 100, args = {
'obs_names': None, 'var_names': None, 'diffexp_lfc_cutoff': 0.01, "layout": "umap",
'nan_to_num': True} "diffexp": "ttest",
"max_category_items": 100,
"obs_names": None,
"var_names": None,
"diffexp_lfc_cutoff": 0.01,
"nan_to_num": True,
}
self.data = ScanpyEngine("example-dataset/pbmc3k.h5ad", args) self.data = ScanpyEngine("example-dataset/pbmc3k.h5ad", args)
self.data._create_schema() self.data._create_schema()
@@ -23,8 +28,8 @@ class UtilTest(unittest.TestCase):
def test_init(self): def test_init(self):
self.assertEqual(self.data.cell_count, 2638) self.assertEqual(self.data.cell_count, 2638)
self.assertEqual(self.data.gene_count, 1838) self.assertEqual(self.data.gene_count, 1838)
epsilon = 0.000005 epsilon = 0.000_005
self.assertTrue(self.data.data.X[0, 0] - -0.17146951 < epsilon) self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
def test_mandatory_annotations(self): def test_mandatory_annotations(self):
self.assertIn("name", self.data.data.obs) self.assertIn("name", self.data.data.obs)
@@ -39,69 +44,36 @@ class UtilTest(unittest.TestCase):
self.data._validate_data_types() self.data._validate_data_types()
def test_filter_idx(self): def test_filter_idx(self):
filter_ = { filter_ = {"filter": {"var": {"index": [1, 99, [200, 300]]}, "obs": {"index": [1, 99, [1000, 2000]]}}}
"filter": {
"var": {
"index": [1, 99, [200, 300]]
},
"obs": {
"index": [1, 99, [1000, 2000]]
}
}
}
data = self.data.filter_dataframe(filter_["filter"]) data = self.data.filter_dataframe(filter_["filter"])
self.assertEqual(data.shape, (1002, 102)) self.assertEqual(data.shape, (1002, 102))
def test_filter_annotation(self): def test_filter_annotation(self):
filter_ = { filter_ = {
"filter": { "filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["NK cells", "CD8 T cells"]}]}}
"obs": {
"annotation_value": [
{"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
]
}
}
} }
data = self.data.filter_dataframe(filter_["filter"]) data = self.data.filter_dataframe(filter_["filter"])
self.assertEqual(data.shape, (470, 1838)) self.assertEqual(data.shape, (470, 1838))
filter_ = { filter_ = {"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}}
"filter": {
"obs": {
"annotation_value": [
{"name": "n_counts", "min": 3000},
]
}
}
}
data = self.data.filter_dataframe(filter_["filter"]) data = self.data.filter_dataframe(filter_["filter"])
self.assertEqual(data.shape, (497, 1838)) self.assertEqual(data.shape, (497, 1838))
def test_filter_annotation_no_uns(self): def test_filter_annotation_no_uns(self):
filter_ = { filter_ = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}}}
"filter": {
"var": {
"annotation_value": [
{"name": "name", "values": ["RER1"]},
]
}
}
}
data = self.data.filter_dataframe(filter_["filter"]) data = self.data.filter_dataframe(filter_["filter"])
self.assertEqual(data.shape[1], 1) self.assertEqual(data.shape[1], 1)
def test_filter_complex(self): def test_filter_complex(self):
filter_ = { filter_ = {
"filter": { "filter": {
"var": { "var": {"index": [1, 99, [200, 300]]},
"index": [1, 99, [200, 300]]
},
"obs": { "obs": {
"annotation_value": [ "annotation_value": [
{"name": "louvain", "values": ["NK cells", "CD8 T cells"]}, {"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
{"name": "n_counts", "min": 3000}, {"name": "n_counts", "min": 3000},
], ],
"index": [1, 99, [1000, 2000]] "index": [1, 99, [1000, 2000]],
} },
} }
} }
data = self.data.filter_dataframe(filter_["filter"]) data = self.data.filter_dataframe(filter_["filter"])
@@ -117,13 +89,14 @@ class UtilTest(unittest.TestCase):
self.assertEqual(self.data.schema, schema) self.assertEqual(self.data.schema, schema)
def test_schema_produces_error(self): def test_schema_produces_error(self):
self.data.data.obs["time"] = Series(list([time.time() for i in range(self.data.cell_count)]), self.data.data.obs["time"] = Series(
dtype="datetime64[ns]") list([time.time() for i in range(self.data.cell_count)]), dtype="datetime64[ns]"
)
with pytest.raises(TypeError): with pytest.raises(TypeError):
self.data._create_schema() self.data._create_schema()
def test_config(self): def test_config(self):
self.assertEqual(self.data.features["layout"]["obs"], {'available': True, 'interactiveLimit': 50000}) self.assertEqual(self.data.features["layout"]["obs"], {"available": True, "interactiveLimit": 50000})
def test_layout(self): def test_layout(self):
layout = self.data.layout(None) layout = self.data.layout(None)
@@ -153,16 +126,8 @@ class UtilTest(unittest.TestCase):
def test_filtered_annotation(self): def test_filtered_annotation(self):
filter_ = { filter_ = {
"filter": { "filter": {
"obs": { "obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]},
"annotation_value": [ "var": {"annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}]},
{"name": "n_counts", "min": 3000},
]
},
"var": {
"annotation_value": [
{"name": "name", "values": ["ATAD3C", "RER1"]},
]
}
} }
} }
annotations = self.data.annotation(filter_["filter"], "obs") annotations = self.data.annotation(filter_["filter"], "obs")
@@ -173,33 +138,13 @@ class UtilTest(unittest.TestCase):
self.assertEqual(len(annotations["data"]), 2) self.assertEqual(len(annotations["data"]), 2)
def test_filtered_layout(self): def test_filtered_layout(self):
filter_ = { filter_ = {"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}}
"filter": {
"obs": {
"annotation_value": [
{"name": "n_counts", "min": 3000},
]
}
}
}
layout = self.data.layout(filter_["filter"]) layout = self.data.layout(filter_["filter"])
self.assertEqual(len(layout["coordinates"]), 497) self.assertEqual(len(layout["coordinates"]), 497)
def test_diffexp_topN(self): def test_diffexp_topN(self):
f1 = { f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
"filter": { f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
"obs": {
"index": [[0, 500]]
}
}
}
f2 = {
"filter": {
"obs": {
"index": [[500, 1000]]
}
}
}
result = self.data.diffexp_topN(f1["filter"], f2["filter"]) result = self.data.diffexp_topN(f1["filter"], f2["filter"])
self.assertEqual(len(result), 10) self.assertEqual(len(result), 10)
result = self.data.diffexp_topN(f1["filter"], f2["filter"], 20) result = self.data.diffexp_topN(f1["filter"], f2["filter"], 20)
@@ -214,15 +159,7 @@ class UtilTest(unittest.TestCase):
self.assertEqual(len(data_frame_var["obs"]), 2638) self.assertEqual(len(data_frame_var["obs"]), 2638)
def test_filtered_data_frame(self): def test_filtered_data_frame(self):
filter_ = { filter_ = {"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}}
"filter": {
"obs": {
"annotation_value": [
{"name": "n_counts", "min": 3000},
]
}
}
}
data_frame_obs = self.data.data_frame(filter_["filter"], "obs") data_frame_obs = self.data.data_frame(filter_["filter"], "obs")
self.assertEqual(len(data_frame_obs["var"]), 1838) self.assertEqual(len(data_frame_obs["var"]), 1838)
self.assertEqual(len(data_frame_obs["obs"]), 497) self.assertEqual(len(data_frame_obs["obs"]), 497)
@@ -236,15 +173,7 @@ class UtilTest(unittest.TestCase):
def test_data_single_gene(self): def test_data_single_gene(self):
for axis in ["obs", "var"]: for axis in ["obs", "var"]:
filter_ = { filter_ = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}}}
"filter": {
"var": {
"annotation_value": [
{"name": "name", "values": ["RER1"]},
]
}
}
}
data_frame_var = self.data.data_frame(filter_["filter"], axis) data_frame_var = self.data.data_frame(filter_["filter"], axis)
if axis == "obs": if axis == "obs":
self.assertEqual(type(data_frame_var["var"][0]), int) self.assertEqual(type(data_frame_var["var"][0]), int)
@@ -253,5 +182,5 @@ class UtilTest(unittest.TestCase):
self.assertEqual(type(data_frame_var["obs"][0]), int) self.assertEqual(type(data_frame_var["obs"][0]), int)
self.assertIsInstance(data_frame_var["var"][0], (list, tuple)) self.assertIsInstance(data_frame_var["var"][0], (list, tuple))
if __name__ == '__main__': if __name__ == "__main__":
unittest.main() unittest.main()
setup.cfg
+1
View File
@@ -1,2 +1,3 @@
[flake8] [flake8]
max-line-length = 120 max-line-length = 120
ignore = E203