fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-29 23:48:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Black -- formatter for python (#508)

Browse Source 
* Add black

* use black to format code

* Black version
This commit is contained in:
Charlotte Weaver
2018-12-12 09:44:47 -08:00
committed by GitHub
parent a847951658
commit 83154577e4
18 changed files with 636 additions and 791 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
+2 -2
View File
@@ -14,8 +14,8 @@ install:
- docker build .
script:
- set -eo pipefail
- flake8 server/app/
- flake8 server/cli/
- flake8 server
- black --check
- npm run --prefix client/ build
- npm run --prefix client/ test
- pytest -s server/test
server/__main__.py
+5 -2
View File
@@ -2,11 +2,14 @@
if __package__ is None:
import sys
from pathlib import Path
PKG_PATH = Path(__file__).parent
sys.path.insert(0, str(PKG_PATH.parent))
import server
import server # noqa F401
__package__ = PKG_PATH.name
# Main thing
from .cli.cli import cli
from .cli.cli import cli # noqa F402
cli()
server/app/app.py
+10 -6
View File
@@ -14,16 +14,14 @@ REACTIVE_LIMIT = 1_000_000
app = Flask(__name__, static_folder="web/static")
app.json_encoder = Float32JSONEncoder
cache = Cache(app, config={"CACHE_TYPE": "simple", "CACHE_DEFAULT_TIMEOUT": 860000})
cache = Cache(app, config={"CACHE_TYPE": "simple", "CACHE_DEFAULT_TIMEOUT": 860_000})
Compress(app)
CORS(app)
# Config
SECRET_KEY = os.environ.get("CXG_SECRET_KEY", default="SparkleAndShine")
app.config.update(
SECRET_KEY=SECRET_KEY,
)
app.config.update(SECRET_KEY=SECRET_KEY)
# Application Data
data = None
@@ -36,7 +34,13 @@ docs.append(resources.get_swagger_doc())
app.register_blueprint(webapp.bp)
app.register_blueprint(resources.blueprint)
app.register_blueprint(
get_swagger_blueprint(docs, "/api/swagger", produces=["application/json"], title="cellxgene rest api",
description="An API connecting ExpressionMatrix2 clustering algorithm to cellxgene"))
get_swagger_blueprint(
docs,
"/api/swagger",
produces=["application/json"],
title="cellxgene rest api",
description="An API connecting ExpressionMatrix2 clustering algorithm to cellxgene",
)
)
app.add_url_rule("/", endpoint="index")
server/app/driver/driver.py
+2 -6
View File
@@ -11,7 +11,6 @@ Sort order for methods
class CXGDriver(metaclass=ABCMeta):
def __init__(self, data, args):
self.data = self._load_data(data)
self.layout_method = args["layout"]
@@ -24,11 +23,8 @@ class CXGDriver(metaclass=ABCMeta):
def features(self):
features = {
"cluster": {"available": False},
"layout": {
"obs": {"available": False},
"var": {"available": False},
},
"diffexp": {"available": False}
"layout": {"obs": {"available": False}, "var": {"available": False}},
"diffexp": {"available": False},
}
# TODO - Interactive limit should be generated from the actual available methods see GH issue #94
if self.layout_method:
server/app/rest_api/rest.py
+327 -426
View File
@@ -2,9 +2,7 @@ from http import HTTPStatus
import pkg_resources
import warnings
from flask import (
Blueprint, current_app, jsonify, make_response, request
)
from flask import Blueprint, current_app, jsonify, make_response, request
from flask_restful_swagger_2 import Api, swagger, Resource
from werkzeug.datastructures import ImmutableMultiDict
@@ -23,83 +21,78 @@ Sort order for routes
class SchemaAPI(Resource):
@swagger.doc({
"summary": "get schema for dataframe and annotations",
"tags": ["initialize"],
"parameters": [],
"responses": {
"200": {
"description": "schema",
"examples": {
"application/json": {
"schema": {
"dataframe": {
"nObs": 383,
"nVar": 19944,
"type": "float32"
},
"annotations": {
"obs": [
{"name": "name", "type": "string"},
{"name": "tissue_type", "type": "string"},
{"name": "num_reads", "type": "int32"},
{"name": "sample_name", "type": "string"},
{
"name": "clusters",
"type": "categorical",
"categories": [99, 1, "unknown cluster"]
},
{"name": "QScore", "type": "float32"}
],
"var": [
{"name": "name", "type": "string"},
{"name": "gene", "type": "string"}
]
@swagger.doc(
{
"summary": "get schema for dataframe and annotations",
"tags": ["initialize"],
"parameters": [],
"responses": {
"200": {
"description": "schema",
"examples": {
"application/json": {
"schema": {
"dataframe": {"nObs": 383, "nVar": 19944, "type": "float32"},
"annotations": {
"obs": [
{"name": "name", "type": "string"},
{"name": "tissue_type", "type": "string"},
{"name": "num_reads", "type": "int32"},
{"name": "sample_name", "type": "string"},
{
"name": "clusters",
"type": "categorical",
"categories": [99, 1, "unknown cluster"],
},
{"name": "QScore", "type": "float32"},
],
"var": [{"name": "name", "type": "string"}, {"name": "gene", "type": "string"}],
},
}
}
}
},
}
}
},
}
})
)
def get(self):
return make_response(jsonify({"schema": current_app.data.schema}), HTTPStatus.OK)
class ConfigAPI(Resource):
@swagger.doc({
"summary": "Configuration information to assist in front-end adaptation"
" to underlying engine, available functionality, interactive time limits, etc",
"tags": ["initialize"],
"parameters": [],
"responses": {
"200": {
"description": "schema",
"examples": {
"application/json": {
"config": {
"features": [
{"method": "POST", "path": "/cluster/", "available": False},
{
"method": "POST",
"path": "/layout/obs",
"available": True,
"interactiveLimit": 10000
@swagger.doc(
{
"summary": "Configuration information to assist in front-end adaptation"
" to underlying engine, available functionality, interactive time limits, etc",
"tags": ["initialize"],
"parameters": [],
"responses": {
"200": {
"description": "schema",
"examples": {
"application/json": {
"config": {
"features": [
{"method": "POST", "path": "/cluster/", "available": False},
{
"method": "POST",
"path": "/layout/obs",
"available": True,
"interactiveLimit": 10000,
},
{"method": "POST", "path": "/layout/var", "available": False},
],
"displayNames": {
"engine": "ScanPy version 1.33",
"dataset": "/home/joe/mouse/blorth.csv",
},
{"method": "POST", "path": "/layout/var", "available": False}
],
"displayNames": {
"engine": "ScanPy version 1.33",
"dataset": "/home/joe/mouse/blorth.csv"
},
}
}
}
},
}
}
},
}
})
)
def get(self):
config = {
"config": {
@@ -111,50 +104,48 @@ class ConfigAPI(Resource):
],
"displayNames": {
"engine": f"cellxgene Scanpy engine version {pkg_resources.get_distribution('cellxgene').version}",
"dataset": current_app.config["DATASET_TITLE"]
"dataset": current_app.config["DATASET_TITLE"],
},
"parameters": {
"max_category_items": current_app.data.max_category_items
}
"parameters": {"max_category_items": current_app.data.max_category_items},
}
}
return make_response(jsonify(config), HTTPStatus.OK)
class AnnotationsObsAPI(Resource):
@swagger.doc({
"summary": "Fetch annotations (metadata) for all observations.",
"tags": ["annotations"],
"parameters": [{
"in": "query",
"name": "annotation-name",
"type": "string",
"description": "list of 1 or more annotation names"
}],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": [
"tissue_type", "sex", "num_reads", "clusters"
],
"data": [
[0, "lung", "F", 39844, 99],
[1, "heart", "M", 83, 1],
[49, "spleen", None, 2, "unknown cluster"],
]
}
@swagger.doc(
{
"summary": "Fetch annotations (metadata) for all observations.",
"tags": ["annotations"],
"parameters": [
{
"in": "query",
"name": "annotation-name",
"type": "string",
"description": "list of 1 or more annotation names",
}
],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": ["tissue_type", "sex", "num_reads", "clusters"],
"data": [
[0, "lung", "F", 39844, 99],
[1, "heart", "M", 83, 1],
[49, "spleen", None, 2, "unknown cluster"],
],
}
},
},
"400": {
"description": "one or more of the annotation-name identifiers were not associated with an "
"annotation name"
},
},
"400": {
"description": "one or more of the annotation-name identifiers were not associated with an "
"annotation name"
}
}
})
)
def get(self):
fields = request.args.getlist("annotation-name", None)
try:
@@ -168,47 +159,40 @@ class AnnotationsObsAPI(Resource):
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
@swagger.doc({
"summary": "Fetch annotations (metadata) for filtered subset of observations.",
"tags": ["annotations"],
"parameters": [
{
"in": "query",
"name": "annotation-name",
"type": "string",
"description": "list of 1 or more annotation names"
@swagger.doc(
{
"summary": "Fetch annotations (metadata) for filtered subset of observations.",
"tags": ["annotations"],
"parameters": [
{
"in": "query",
"name": "annotation-name",
"type": "string",
"description": "list of 1 or more annotation names",
},
{"name": "filter", "description": "Complex Filter", "in": "body", "schema": FilterModel},
],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": ["tissue_type", "sex", "num_reads", "clusters"],
"data": [
[0, "lung", "F", 39844, 99],
[1, "heart", "M", 83, 1],
[49, "spleen", None, 2, "unknown cluster"],
],
}
},
},
"400": {
"description": "malformed filter or one or more of the annotation-name identifiers were"
"not associated with an annotation name"
},
},
{
"name": "filter",
"description": "Complex Filter",
"in": "body",
"schema": FilterModel
}
],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": [
"tissue_type", "sex", "num_reads", "clusters"
],
"data": [
[0, "lung", "F", 39844, 99],
[1, "heart", "M", 83, 1],
[49, "spleen", None, 2, "unknown cluster"],
]
}
}
},
"400": {
"description": "malformed filter or one or more of the annotation-name identifiers were"
"not associated with an annotation name"
}
}
})
)
def put(self):
fields = request.args.getlist("annotation-name", None)
try:
@@ -226,38 +210,35 @@ class AnnotationsObsAPI(Resource):
class AnnotationsVarAPI(Resource):
@swagger.doc({
"summary": "Fetch annotations (metadata) for all variables.",
"tags": ["annotations"],
"parameters": [{
"in": "query",
"name": "annotation-name",
"type": "string",
"description": "list of 1 or more annotation names"
}],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": [
"name", "category"
],
"data": [
[0, "ATAD3C", 1],
[1, "RER1", None],
[49, "S100B", 6]
]
}
@swagger.doc(
{
"summary": "Fetch annotations (metadata) for all variables.",
"tags": ["annotations"],
"parameters": [
{
"in": "query",
"name": "annotation-name",
"type": "string",
"description": "list of 1 or more annotation names",
}
],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": ["name", "category"],
"data": [[0, "ATAD3C", 1], [1, "RER1", None], [49, "S100B", 6]],
}
},
},
"400": {
"description": "one or more of the annotation-name identifiers were not associated with an"
" annotation name"
},
},
"400": {
"description": "one or more of the annotation-name identifiers were not associated with an"
" annotation name"
}
}
})
)
def get(self):
fields = request.args.getlist("annotation-name", None)
try:
@@ -271,45 +252,36 @@ class AnnotationsVarAPI(Resource):
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
@swagger.doc({
"summary": "Fetch annotations (metadata) for filtered subset of variables.",
"tags": ["annotations"],
"parameters": [
{
"in": "query",
"name": "annotation-name",
"type": "string",
"description": "list of 1 or more annotation names"
@swagger.doc(
{
"summary": "Fetch annotations (metadata) for filtered subset of variables.",
"tags": ["annotations"],
"parameters": [
{
"in": "query",
"name": "annotation-name",
"type": "string",
"description": "list of 1 or more annotation names",
},
{"name": "filter", "description": "Complex Filter", "in": "body", "schema": FilterModel},
],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": ["name", "category"],
"data": [[0, "ATAD3C", 1], [1, "RER1", None], [49, "S100B", 6]],
}
},
},
"400": {
"description": "malformed filter or one or more of the annotation-name identifiers were"
"not associated with an annotation name"
},
},
{
"name": "filter",
"description": "Complex Filter",
"in": "body",
"schema": FilterModel
}
],
"responses": {
"200": {
"description": "annotations",
"examples": {
"application/json": {
"names": [
"name", "category"
],
"data": [
[0, "ATAD3C", 1],
[1, "RER1", None],
[49, "S100B", 6]
]
}
}
},
"400": {
"description": "malformed filter or one or more of the annotation-name identifiers were"
"not associated with an annotation name"
}
}
})
)
def put(self):
fields = request.args.getlist("annotation-name", None)
try:
@@ -327,57 +299,39 @@ class AnnotationsVarAPI(Resource):
class DataObsAPI(Resource):
@swagger.doc({
"summary": "Get data (expression values) from the dataframe.",
"tags": ["data"],
"parameters": [
{
"in": "query",
"name": "filter",
"type": "string",
"description": "axis:key:value"
},
{
"in": "query",
"name": "accept-type",
"type": "string",
"description": "MIME type"
},
],
"responses": {
"200": {
"description": "expression",
"examples": {
"application/json": {
"var": [0, 20000],
"obs": [
[1, 39483, 3902, 203, 0, 0, 28]
]
}
}
},
"400": {
"description": "Malformed filter"
},
"406": {
"description": "Unacceptable MIME type"
@swagger.doc(
{
"summary": "Get data (expression values) from the dataframe.",
"tags": ["data"],
"parameters": [
{"in": "query", "name": "filter", "type": "string", "description": "axis:key:value"},
{"in": "query", "name": "accept-type", "type": "string", "description": "MIME type"},
],
"responses": {
"200": {
"description": "expression",
"examples": {"application/json": {"var": [0, 20000], "obs": [[1, 39483, 3902, 203, 0, 0, 28]]}},
},
"400": {"description": "Malformed filter"},
"406": {"description": "Unacceptable MIME type"},
},
}
})
)
def get(self):
accept_type = request.args.get("accept-type", None)
# request.args is immutable
args = request.args.copy()
args.pop("accept-type", None)
try:
filter_ = parse_filter(ImmutableMultiDict(args), current_app.data.schema['annotations'])
filter_ = parse_filter(ImmutableMultiDict(args), current_app.data.schema["annotations"])
except QueryStringError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
# TODO support CSV
try:
# TODO store mime_type when more than one is supported
get_mime_type(acceptable_types=["application/json"], query_param=accept_type,
header=request.accept_mimetypes)
get_mime_type(
acceptable_types=["application/json"], query_param=accept_type, header=request.accept_mimetypes
)
except MimeTypeError as e:
return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
try:
@@ -389,37 +343,21 @@ class DataObsAPI(Resource):
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
@swagger.doc({
"summary": "Get data (expression values) from the dataframe.",
"tags": ["data"],
"parameters": [
{
'name': 'filter',
'description': 'Complex Filter',
'in': 'body',
'schema': FilterModel
}
],
"responses": {
"200": {
"description": "expression",
"examples": {
"application/json": {
"var": [0, 20000],
"obs": [
[1, 39483, 3902, 203, 0, 0, 28]
]
}
}
},
"400": {
"description": "Malformed filter"
},
"406": {
"description": "Unacceptable MIME type"
@swagger.doc(
{
"summary": "Get data (expression values) from the dataframe.",
"tags": ["data"],
"parameters": [{"name": "filter", "description": "Complex Filter", "in": "body", "schema": FilterModel}],
"responses": {
"200": {
"description": "expression",
"examples": {"application/json": {"var": [0, 20000], "obs": [[1, 39483, 3902, 203, 0, 0, 28]]}},
},
"400": {"description": "Malformed filter"},
"406": {"description": "Unacceptable MIME type"},
},
}
})
)
def put(self):
if not request.accept_mimetypes.best_match(["application/json", "text/csv"]):
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
@@ -428,8 +366,9 @@ class DataObsAPI(Resource):
except MimeTypeError as e:
return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
try:
return make_response((jsonify(current_app.data.data_frame(request.get_json()["filter"], axis=Axis.OBS))),
HTTPStatus.OK)
return make_response(
(jsonify(current_app.data.data_frame(request.get_json()["filter"], axis=Axis.OBS))), HTTPStatus.OK
)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except ValueError as e:
@@ -439,55 +378,37 @@ class DataObsAPI(Resource):
class DataVarAPI(Resource):
@swagger.doc({
"summary": "Get data (expression values) from the dataframe.",
"tags": ["data"],
"parameters": [
{
"in": "query",
"name": "filter",
"type": "string",
"description": "axis:key:value"
},
{
"in": "query",
"name": "accept-type",
"type": "string",
"description": "MIME type"
},
],
"responses": {
"200": {
"description": "expression",
"examples": {
"application/json": {
"obs": [0, 20000],
"var": [
[1, 39483, 3902, 203, 0, 0, 28]
]
}
}
},
"400": {
"description": "Malformed filter"
},
"406": {
"description": "Unacceptable MIME type"
@swagger.doc(
{
"summary": "Get data (expression values) from the dataframe.",
"tags": ["data"],
"parameters": [
{"in": "query", "name": "filter", "type": "string", "description": "axis:key:value"},
{"in": "query", "name": "accept-type", "type": "string", "description": "MIME type"},
],
"responses": {
"200": {
"description": "expression",
"examples": {"application/json": {"obs": [0, 20000], "var": [[1, 39483, 3902, 203, 0, 0, 28]]}},
},
"400": {"description": "Malformed filter"},
"406": {"description": "Unacceptable MIME type"},
},
}
})
)
def get(self):
accept_type = request.args.get("accept-type", None)
# request.args is immutable
args = request.args.copy()
args.pop("accept-type", None)
try:
filter_ = parse_filter(ImmutableMultiDict(args), current_app.data.schema['annotations'])
filter_ = parse_filter(ImmutableMultiDict(args), current_app.data.schema["annotations"])
except QueryStringError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
try:
get_mime_type(acceptable_types=["application/json"], query_param=accept_type,
header=request.accept_mimetypes)
get_mime_type(
acceptable_types=["application/json"], query_param=accept_type, header=request.accept_mimetypes
)
except MimeTypeError as e:
return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
try:
@@ -499,37 +420,21 @@ class DataVarAPI(Resource):
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
@swagger.doc({
"summary": "Get data (expression values) from the dataframe.",
"tags": ["data"],
"parameters": [
{
'name': 'filter',
'description': 'Complex Filter',
'in': 'body',
'schema': FilterModel
}
],
"responses": {
"200": {
"description": "expression",
"examples": {
"application/json": {
"obs": [0, 20000],
"var": [
[1, 39483, 3902, 203, 0, 0, 28]
]
}
}
},
"400": {
"description": "Malformed filter"
},
"406": {
"description": "Unacceptable MIME type"
@swagger.doc(
{
"summary": "Get data (expression values) from the dataframe.",
"tags": ["data"],
"parameters": [{"name": "filter", "description": "Complex Filter", "in": "body", "schema": FilterModel}],
"responses": {
"200": {
"description": "expression",
"examples": {"application/json": {"obs": [0, 20000], "var": [[1, 39483, 3902, 203, 0, 0, 28]]}},
},
"400": {"description": "Malformed filter"},
"406": {"description": "Unacceptable MIME type"},
},
}
})
)
def put(self):
if not request.accept_mimetypes.best_match(["application/json", "text/csv"]):
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
@@ -539,8 +444,9 @@ class DataVarAPI(Resource):
except MimeTypeError as e:
return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
try:
return make_response((jsonify(current_app.data.data_frame(request.get_json()["filter"], axis=Axis.VAR))),
HTTPStatus.OK)
return make_response(
(jsonify(current_app.data.data_frame(request.get_json()["filter"], axis=Axis.VAR))), HTTPStatus.OK
)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except ValueError as e:
@@ -550,67 +456,64 @@ class DataVarAPI(Resource):
class DiffExpObsAPI(Resource):
@swagger.doc({
"summary": "Generate differential expression (DE) statistics for two specified subsets of data, "
"as indicated by the two provided observation complex filters",
"tags": ["diffexp"],
# TODO sort out params
# "parameters": [
# # {
# # "in": "body",
# # "name": "mode",
# # "type": "string",
# # "required": True,
# # "description": "topN or varFilter"
# # },
# {
# "in": "query",
# "name": "count",
# "type": "int32",
# "description": "TopN mode: how many vars to return"
# },
# {
# "in": "body",
# "name": "varFilter",
# "schema": FilterModel,
# "description": "varFilter: Complex filter, only var for which vars to return"
# },
# {
# "in": "body",
# "name": "set1",
# "schema": FilterModel,
# "required": True,
# "description": "Complex filter, only obs - observations in set1"
# },
# {
# "in": "body",
# "name": "set2",
# "schema": FilterModel,
# "description": "Complex filter, only obs - observations in set2. If not included, inverse of set1."
# },
# ],
"responses": {
"200": {
"description": "Statistics are encoded as an array of arrays, with fields ordered as: "
"varIndex, logfoldchange, pVal, pValAdj",
"examples": {
"application/json": [
[328, -2.569489, 2.655706e-63, 3.642036e-57],
[1250, -2.569489, 2.655706e-63, 3.642036e-57],
]
}
@swagger.doc(
{
"summary": "Generate differential expression (DE) statistics for two specified subsets of data, "
"as indicated by the two provided observation complex filters",
"tags": ["diffexp"],
# TODO sort out params
# "parameters": [
# # {
# # "in": "body",
# # "name": "mode",
# # "type": "string",
# # "required": True,
# # "description": "topN or varFilter"
# # },
# {
# "in": "query",
# "name": "count",
# "type": "int32",
# "description": "TopN mode: how many vars to return"
# },
# {
# "in": "body",
# "name": "varFilter",
# "schema": FilterModel,
# "description": "varFilter: Complex filter, only var for which vars to return"
# },
# {
# "in": "body",
# "name": "set1",
# "schema": FilterModel,
# "required": True,
# "description": "Complex filter, only obs - observations in set1"
# },
# {
# "in": "body",
# "name": "set2",
# "schema": FilterModel,
# "description": "Complex filter, only obs - observations in set2.
# If not included, inverse of set1."
# },
# ],
"responses": {
"200": {
"description": "Statistics are encoded as an array of arrays, with fields ordered as: "
"varIndex, logfoldchange, pVal, pValAdj",
"examples": {
"application/json": [
[328, -2.569_489, 2.655_706e-63, 3.642_036e-57],
[1250, -2.569_489, 2.655_706e-63, 3.642_036e-57],
]
},
},
"400": {"description": "malformed filter"},
"403": {"description": "non-interactive request"},
"501": {"description": "diffexp is not implemented"},
},
"400": {
"description": "malformed filter"
},
"403": {
"description": "non-interactive request"
},
"501": {
"description": "diffexp is not implemented"
}
}
})
)
def post(self):
args = request.get_json()
# confirm mode is present and legal
@@ -645,8 +548,9 @@ class DiffExpObsAPI(Resource):
# mode=topN
count = args.get("count", None)
try:
diffexp = current_app.data.diffexp_topN(set1_filter, set2_filter, count,
current_app.data.features["diffexp"]["interactiveLimit"])
diffexp = current_app.data.diffexp_topN(
set1_filter, set2_filter, count, current_app.data.features["diffexp"]["interactiveLimit"]
)
except (ValueError, FilterError) as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except InteractiveError:
@@ -660,30 +564,27 @@ class DiffExpObsAPI(Resource):
class LayoutObsAPI(Resource):
@swagger.doc({
"summary": "Get the default layout for all observations.",
"tags": ["layout"],
"parameters": [],
"responses": {
"200": {
"description": "layout",
"examples": {
"application/json": {
"layout": {
"ndims": 2,
"coordinates": [
[0, 0.284483, 0.983744],
[1, 0.038844, 0.739444]
]
@swagger.doc(
{
"summary": "Get the default layout for all observations.",
"tags": ["layout"],
"parameters": [],
"responses": {
"200": {
"description": "layout",
"examples": {
"application/json": {
"layout": {
"ndims": 2,
"coordinates": [[0, 0.284_483, 0.983_744], [1, 0.038_844, 0.739_444]],
}
}
}
}
},
},
"400": {"description": "Data preparation error"},
},
"400": {
"description": "Data preparation error"
}
}
})
)
def get(self):
try:
layout = current_app.data.layout({})
server/app/scanpy_engine/diffexp.py
+5 -9
View File
@@ -1,4 +1,3 @@
import numpy as np
from scipy import sparse, stats
@@ -64,19 +63,19 @@ def diffexp_ttest(adata, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
sum_vn = vnA + vnB
# degrees of freedom for Welch's t-test
with np.errstate(divide='ignore', invalid='ignore'):
dof = sum_vn**2 / (vnA**2 / (nA - 1) + vnB**2 / (nB - 1))
with np.errstate(divide="ignore", invalid="ignore"):
dof = sum_vn ** 2 / (vnA ** 2 / (nA - 1) + vnB ** 2 / (nB - 1))
dof[np.isnan(dof)] = 1
# Welch's t-test score calculation
with np.errstate(divide='ignore', invalid='ignore'):
with np.errstate(divide="ignore", invalid="ignore"):
tscores = (meanA - meanB) / np.sqrt(sum_vn)
tscores[np.isnan(tscores)] = 0
# p-value
pvals = stats.t.sf(np.abs(tscores), dof) * 2
pvals_adj = pvals * adata._X.shape[1]
pvals_adj[pvals_adj > 1] = 1 # cap adjusted p-value at 1
pvals_adj[pvals_adj > 1] = 1 # cap adjusted p-value at 1
# logfoldchanges: log2(meanA / meanB)
logfoldchanges = np.log2(np.abs((meanA + 1e-9) / (meanB + 1e-9)))
@@ -106,8 +105,5 @@ def diffexp_ttest(adata, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
pvals_adj_top_n = pvals_adj[sort_order]
# varIndex, logfoldchange, pval, pval_adj
result = [[sort_order[i],
logfoldchanges_top_n[i],
pvals_top_n[i],
pvals_adj_top_n[i]] for i in range(top_n)]
result = [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n)]
return result
server/app/scanpy_engine/scanpy_engine.py
+53 -62
View File
@@ -22,7 +22,6 @@ Sort order for methods
class ScanpyEngine(CXGDriver):
def __init__(self, data, args):
super().__init__(data, args)
self._alias_annotation_names(Axis.OBS, args["obs_names"])
@@ -36,7 +35,7 @@ class ScanpyEngine(CXGDriver):
self._create_schema()
# TODO: temporary work-arounds
if args['nan_to_num']:
if args["nan_to_num"]:
self._IEEE754_special_values_workaround()
def _alias_annotation_names(self, axis, name):
@@ -61,8 +60,9 @@ class ScanpyEngine(CXGDriver):
if name not in df_axis.columns:
raise KeyError(f"Annotation name {name}, specified in --{ax_name}-name does not exist.")
if not df_axis[name].is_unique:
raise KeyError(f"Values in -{ax_name}-name must be unique. "
"Please prepare data to contain unique values.")
raise KeyError(
f"Values in -{ax_name}-name must be unique. " "Please prepare data to contain unique values."
)
# reset index to simple range; alias user-specified annotation to "name"
df_axis.reset_index(drop=True, inplace=True)
df_axis.rename(inplace=True, columns={name: "name"})
@@ -89,15 +89,8 @@ class ScanpyEngine(CXGDriver):
def _create_schema(self):
self.schema = {
"dataframe": {
"nObs": self.cell_count,
"nVar": self.gene_count,
"type": str(self.data.X.dtype)
},
"annotations": {
"obs": [],
"var": []
}
"dataframe": {"nObs": self.cell_count, "nVar": self.gene_count, "type": str(self.data.X.dtype)},
"annotations": {"obs": [], "var": []},
}
for ax in Axis:
curr_axis = getattr(self.data, str(ax))
@@ -129,32 +122,35 @@ class ScanpyEngine(CXGDriver):
try:
result = sc.read(data, cache=True)
except ValueError:
raise ScanpyFileError("File must be in the .h5ad format. Please read "
"https://github.com/theislab/scanpy_usage/blob/master/170505_seurat/info_h5ad.md to "
"learn more about this format. You may be able to convert your file into this format "
"using `cellxgene prepare`, please run `cellxgene prepare --help` for more "
"information.")
raise ScanpyFileError(
"File must be in the .h5ad format. Please read "
"https://github.com/theislab/scanpy_usage/blob/master/170505_seurat/info_h5ad.md to "
"learn more about this format. You may be able to convert your file into this format "
"using `cellxgene prepare`, please run `cellxgene prepare --help` for more "
"information."
)
except Exception as e:
raise ScanpyFileError(f"Error while loading file: {e}, File must be in the .h5ad format, please check "
f"that your input and try again.")
raise ScanpyFileError(
f"Error while loading file: {e}, File must be in the .h5ad format, please check "
f"that your input and try again."
)
return result
def _validate_data_types(self):
if self.data.X.dtype != "float32":
warnings.warn(f"Scanpy data matrix is in {self.data.X.dtype} format not float32. "
f"Precision may be truncated.")
warnings.warn(
f"Scanpy data matrix is in {self.data.X.dtype} format not float32. " f"Precision may be truncated."
)
for ax in Axis:
curr_axis = getattr(self.data, str(ax))
for ann in curr_axis:
datatype = curr_axis[ann].dtype
downcast_map = {"int64": "int32",
"uint32": "int32",
"uint64": "int32",
"float64": "float32",
}
downcast_map = {"int64": "int32", "uint32": "int32", "uint64": "int32", "float64": "float32"}
if datatype in downcast_map:
warnings.warn(f"Scanpy annotation {ax}:{ann} is in unsupported format: {datatype}. "
f"Data will be downcast to {downcast_map[datatype]}.")
warnings.warn(
f"Scanpy annotation {ax}:{ann} is in unsupported format: {datatype}. "
f"Data will be downcast to {downcast_map[datatype]}."
)
if isinstance(datatype, CategoricalDtype):
category_num = len(curr_axis[ann].dtype.categories)
if category_num > 500 and category_num > self.max_category_items:
@@ -162,7 +158,8 @@ class ScanpyEngine(CXGDriver):
f"{str(ax).title()} annotation '{ann}' has {category_num} categories, this may be "
f"cumbersome or slow to display. We recommend setting the "
f"--max-category-items option to 500, this will hide categorical "
f"annotations with more than 500 categories in the UI")
f"annotations with more than 500 categories in the UI"
)
def _validate_data_calculations(self):
layout_key = f"X_{self.layout_method}"
@@ -174,7 +171,8 @@ class ScanpyEngine(CXGDriver):
f" layout may have been computed. The requested layout must be pre-calculated and saved "
f"back in the h5ad file. You can run "
f"`cellxgene prepare --layout {self.layout_method} <datafile>` "
f"to solve this problem. ")
f"to solve this problem. "
)
def _IEEE754_special_values_workaround(self):
"""
@@ -196,7 +194,7 @@ class ScanpyEngine(CXGDriver):
curr_axis = getattr(self.data, str(ax))
for ann in curr_axis:
dtype = curr_axis[ann].dtype
if dtype.kind == 'f':
if dtype.kind == "f":
finite_idx = np.isfinite(curr_axis[ann])
if not finite_idx.all():
curr_axis.loc[np.isnan(curr_axis[ann]), ann] = 0
@@ -233,8 +231,7 @@ class ScanpyEngine(CXGDriver):
if non_finite_X_found:
warnings.warn(
"Dataframe X contains floating point NaN or Infinities. "
"These will be converted to finite values."
"Dataframe X contains floating point NaN or Infinities. " "These will be converted to finite values."
)
def filter_dataframe(self, filter):
@@ -256,7 +253,7 @@ class ScanpyEngine(CXGDriver):
@staticmethod
def _annotation_filter_to_mask(filter, d_axis, count):
mask = np.ones((count, ), dtype=bool)
mask = np.ones((count,), dtype=bool)
for v in filter:
if d_axis[v["name"]].dtype.name in ["boolean", "category", "object"]:
key_idx = np.in1d(getattr(d_axis, v["name"]), v["values"])
@@ -274,24 +271,23 @@ class ScanpyEngine(CXGDriver):
@staticmethod
def _index_filter_to_mask(filter, count):
mask = np.zeros((count, ), dtype=bool)
mask = np.zeros((count,), dtype=bool)
for i in filter:
if type(i) == list:
mask[i[0]:i[1]] = True
mask[i[0] : i[1]] = True
else:
mask[i] = True
return mask
@staticmethod
def _axis_filter_to_mask(filter, d_axis, count):
mask = np.ones((count, ), dtype=bool)
mask = np.ones((count,), dtype=bool)
if "index" in filter:
mask = np.logical_and(mask, ScanpyEngine._index_filter_to_mask(filter["index"], count))
if "annotation_value" in filter:
mask = np.logical_and(mask,
ScanpyEngine._annotation_filter_to_mask(filter["annotation_value"],
d_axis,
count))
mask = np.logical_and(
mask, ScanpyEngine._annotation_filter_to_mask(filter["annotation_value"], d_axis, count)
)
return mask
def _filter_to_mask(self, filter, use_slices=True):
@@ -321,8 +317,9 @@ class ScanpyEngine(CXGDriver):
https://docs.scipy.org/doc/scipy/reference/sparse.html
"""
prefer_row_access = sparse.isspmatrix_csr(data._X) or sparse.isspmatrix_lil(data._X) \
or sparse.isspmatrix_bsr(data._X)
prefer_row_access = (
sparse.isspmatrix_csr(data._X) or sparse.isspmatrix_lil(data._X) or sparse.isspmatrix_bsr(data._X)
)
if prefer_row_access:
# Row-major slicing
if obs_selector is not None:
@@ -355,18 +352,12 @@ class ScanpyEngine(CXGDriver):
obs = self.data.obs[obs_selector]
if not fields:
fields = obs.columns.tolist()
result = {
"names": fields,
"data": DataFrame(obs[fields]).to_records(index=True).tolist()
}
result = {"names": fields, "data": DataFrame(obs[fields]).to_records(index=True).tolist()}
else:
var = self.data.var[var_selector]
if not fields:
fields = var.columns.tolist()
result = {
"names": fields,
"data": DataFrame(var[fields]).to_records(index=True).tolist()
}
result = {"names": fields, "data": DataFrame(var[fields]).to_records(index=True).tolist()}
return result
def data_frame(self, filter, axis):
@@ -391,12 +382,12 @@ class ScanpyEngine(CXGDriver):
if axis == Axis.OBS:
result = {
"var": var_index_sliced.tolist(),
"obs": DataFrame(_X, index=obs_index_sliced).to_records(index=True).tolist()
"obs": DataFrame(_X, index=obs_index_sliced).to_records(index=True).tolist(),
}
else:
result = {
"obs": obs_index_sliced.tolist(),
"var": DataFrame(_X.T, index=var_index_sliced).to_records(index=True).tolist()
"var": DataFrame(_X.T, index=var_index_sliced).to_records(index=True).tolist(),
}
return result
@@ -435,11 +426,11 @@ class ScanpyEngine(CXGDriver):
try:
df_layout = df.obsm[f"X_{self.layout_method}"]
except ValueError as e:
raise PrepareError(f"Layout has not been calculated using {self.layout_method}, "
f"please prepare your datafile and relaunch cellxgene") from e
normalized_layout = DataFrame((df_layout - df_layout.min()) / (df_layout.max() - df_layout.min()),
index=df.obs.index)
return {
"ndims": normalized_layout.shape[1],
"coordinates": normalized_layout.to_records(index=True).tolist()
}
raise PrepareError(
f"Layout has not been calculated using {self.layout_method}, "
f"please prepare your datafile and relaunch cellxgene"
) from e
normalized_layout = DataFrame(
(df_layout - df_layout.min()) / (df_layout.max() - df_layout.min()), index=df.obs.index
)
return {"ndims": normalized_layout.shape[1], "coordinates": normalized_layout.to_records(index=True).tolist()}
server/app/util/filter.py
-1
View File
@@ -7,7 +7,6 @@ from server.app.util.constants import Axis
class QueryStringError(Exception):
def __init__(self, key, message):
self.key = key
self.message = message
server/app/util/models.py
+7 -38
View File
@@ -5,24 +5,13 @@ class AnnotationModel(Schema):
type = "object"
description = "Filter by annotation key: value"
properties = {
"name": {
"type": "string"
},
"name": {"type": "string"},
# TODO update to OpenAPI v3.0 when a library is available that supports it
# Unfortunately 2.0 doesn't have a way to have a schema that accepts multiple types
# Overloading the type key with a list seems to work ok and makes it to the page
"values": {
"type": "array",
"items": {
"type": ["float32", "string", "int32", "bool"]
}
},
"min": {
"type": ["int32", "float32"],
},
"max": {
"type": ["int32", "float32"],
}
"values": {"type": "array", "items": {"type": ["float32", "string", "int32", "bool"]}},
"min": {"type": ["int32", "float32"]},
"max": {"type": ["int32", "float32"]},
}
required = ["name"]
@@ -30,36 +19,16 @@ class AnnotationModel(Schema):
class IndexModel(Schema):
type = "object"
description = "Filter by index of observation/variable ex. [0, 5, 15]"
properties = {
"index": {
"type": "array",
"items": {
"format": "int32",
"type": "integer"
}
}
}
properties = {"index": {"type": "array", "items": {"format": "int32", "type": "integer"}}}
class AxisModel(Schema):
type = "object"
description = "Axis of data -- obs or var"
properties = {
"index": IndexModel,
"annotation_value": AnnotationModel.array()
}
properties = {"index": IndexModel, "annotation_value": AnnotationModel.array()}
class FilterModel(Schema):
type = "object"
description = "Complex filter"
properties = {
"filter": {
"type": "object",
"properties": {
"obs": AxisModel,
"var": AxisModel
}
}
}
properties = {"filter": {"type": "object", "properties": {"obs": AxisModel, "var": AxisModel}}}
server/app/util/utils.py
+4 -3
View File
@@ -15,7 +15,7 @@ class Float32JSONEncoder(json.JSONEncoder):
if it runs into non-finite floating point values which are unsupported by
standard JSON.
"""
kwargs['allow_nan'] = False
kwargs["allow_nan"] = False
super().__init__(*args, **kwargs)
def default(self, obj):
@@ -30,8 +30,9 @@ def custom_format_warning(msg, *args, **kwargs):
return f"[cellxgene] Warning: {msg} \n"
def get_mime_type(default="application/json", acceptable_types=["application/json", "text/csv"], query_param=None,
header=None):
def get_mime_type(
default="application/json", acceptable_types=["application/json", "text/csv"], query_param=None, header=None
):
mime_type = default
if query_param:
if query_param in acceptable_types:
server/app/web/webapp.py
+1 -3
View File
@@ -1,7 +1,5 @@
import os
from flask import (
Blueprint, render_template, send_from_directory, current_app
)
from flask import Blueprint, render_template, send_from_directory, current_app
bp = Blueprint("webapp", __name__, template_folder="templates")
server/cli/launch.py
+70 -26
View File
@@ -13,30 +13,76 @@ from server.app.util.utils import custom_format_warning
@click.command()
@click.argument("data", metavar="<data file>", type=click.Path(exists=True, file_okay=True, dir_okay=False))
@click.option("--layout", "-l", type=click.Choice(["umap", "tsne"]), default="umap", show_default=True,
help="Method for layout.")
@click.option("--diffexp", "-d", type=click.Choice(["ttest"]), default="ttest", show_default=True,
help="Method for differential expression.")
@click.option(
"--layout", "-l", type=click.Choice(["umap", "tsne"]), default="umap", show_default=True, help="Method for layout."
)
@click.option(
"--diffexp",
"-d",
type=click.Choice(["ttest"]),
default="ttest",
show_default=True,
help="Method for differential expression.",
)
@click.option("--title", "-t", help="Title to display (if omitted will use file name).", metavar="")
@click.option("--verbose", "-v", is_flag=True, default=False, show_default=True,
help="Provide verbose output, including warnings and all server requests.")
@click.option("--debug", "-d", is_flag=True, default=False, show_default=True,
help="Run in debug mode.")
@click.option("--open", "-o", "open_browser", is_flag=True, default=False, show_default=True,
help="Open the web browser after launch.")
@click.option(
"--verbose",
"-v",
is_flag=True,
default=False,
show_default=True,
help="Provide verbose output, including warnings and all server requests.",
)
@click.option("--debug", "-d", is_flag=True, default=False, show_default=True, help="Run in debug mode.")
@click.option(
"--open",
"-o",
"open_browser",
is_flag=True,
default=False,
show_default=True,
help="Open the web browser after launch.",
)
@click.option("--port", "-p", help="Port to run server on.", metavar="", default=5005, show_default=True)
@click.option("--obs-names", default=None, metavar="", help="Name of annotation field to use for observations.")
@click.option("--var-names", default=None, metavar="", help="Name of annotation to use for variables.")
@click.option("--host", default="127.0.0.1", help="Host IP address")
@click.option("--max-category-items", default=100, metavar="", show_default=True,
help="Limits the number of categorical annotation items displayed.")
@click.option("--diffexp-lfc-cutoff", default=0.01, show_default=True,
help="Relative expression cutoff used when selecting top N differentially expressed genes")
@click.option("--nan-to-num", is_flag=True, default=False, show_default=True,
help="Replace all floating point NaN with zero, and infinities with finite numbers")
def launch(data, layout, diffexp, title, verbose, debug, obs_names, var_names,
open_browser, port, host, max_category_items, diffexp_lfc_cutoff,
nan_to_num):
@click.option(
"--max-category-items",
default=100,
metavar="",
show_default=True,
help="Limits the number of categorical annotation items displayed.",
)
@click.option(
"--diffexp-lfc-cutoff",
default=0.01,
show_default=True,
help="Relative expression cutoff used when selecting top N differentially expressed genes",
)
@click.option(
"--nan-to-num",
is_flag=True,
default=False,
show_default=True,
help="Replace all floating point NaN with zero, and infinities with finite numbers",
)
def launch(
data,
layout,
diffexp,
title,
verbose,
debug,
obs_names,
var_names,
open_browser,
port,
host,
max_category_items,
diffexp_lfc_cutoff,
nan_to_num,
):
"""Launch the cellxgene data viewer.
This web app lets you explore single-cell expression data.
Data must be in a format that cellxgene expects, read the
@@ -76,10 +122,7 @@ def launch(data, layout, diffexp, title, verbose, debug, obs_names, var_names,
# Import Flask app
from server.app.app import app
app.config.update(
DATASET_TITLE=title,
CXG_API_BASE=api_base
)
app.config.update(DATASET_TITLE=title, CXG_API_BASE=api_base)
if not verbose:
log = logging.getLogger("werkzeug")
@@ -90,7 +133,8 @@ def launch(data, layout, diffexp, title, verbose, debug, obs_names, var_names,
# Fix for anaconda python. matplotlib typically expects python to be installed as a framework TKAgg is usually
# available and fixes this issue. See https://matplotlib.org/faq/virtualenv_faq.html
import matplotlib as mpl
mpl.use('TkAgg')
mpl.use("TkAgg")
from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
args = {
@@ -100,7 +144,7 @@ def launch(data, layout, diffexp, title, verbose, debug, obs_names, var_names,
"diffexp_lfc_cutoff": diffexp_lfc_cutoff,
"obs_names": obs_names,
"var_names": var_names,
"nan_to_num": nan_to_num
"nan_to_num": nan_to_num,
}
try:
@@ -117,7 +161,7 @@ def launch(data, layout, diffexp, title, verbose, debug, obs_names, var_names,
click.echo("[cellxgene] Type CTRL-C at any time to exit.")
if not verbose:
f = open(devnull, 'w')
f = open(devnull, "w")
sys.stdout = f
app.run(host=host, debug=debug, port=port, threaded=True)
server/cli/prepare.py
+48 -17
View File
@@ -7,22 +7,48 @@ from scipy.sparse.csc import csc_matrix
@click.command()
@click.argument("data", nargs=1, metavar="<dataset: file or path to data>", required=True)
@click.option("--layout", "-l", default=["umap", "tsne"], multiple=True, type=click.Choice(["umap", "tsne"]),
help="Layout algorithm", show_default=True)
@click.option("--recipe", "-r", default="none", type=click.Choice(["none", "seurat", "zheng17"]),
help="Preprocessing to run.", show_default=True)
@click.option(
"--layout",
"-l",
default=["umap", "tsne"],
multiple=True,
type=click.Choice(["umap", "tsne"]),
help="Layout algorithm",
show_default=True,
)
@click.option(
"--recipe",
"-r",
default="none",
type=click.Choice(["none", "seurat", "zheng17"]),
help="Preprocessing to run.",
show_default=True,
)
@click.option("--output", "-o", default="", help="Save a new file to filename.", metavar="<filename>")
@click.option("--plotting", "-p", default=False, is_flag=True, help="Whether to generate plots.", show_default=True)
@click.option("--sparse", default=False, is_flag=True, help="Whether to force sparsity.", show_default=True)
@click.option("--overwrite", default=False, is_flag=True, help="Allow file overwriting.", show_default=True)
@click.option("--set-obs-names", default="", help="Named field to set as index for obs.", metavar="<name>")
@click.option("--set-var-names", default="", help="Named field to set as index for var.", metavar="<name>")
@click.option("--make-obs-names-unique", default=True, is_flag=True,
help="Ensure obs index is unique.", show_default=True)
@click.option("--make-var-names-unique", default=True, is_flag=True,
help="Ensure var index is unique.", show_default=True)
def prepare(data, layout, recipe, output, plotting, sparse, overwrite,
set_obs_names, set_var_names, make_obs_names_unique, make_var_names_unique):
@click.option(
"--make-obs-names-unique", default=True, is_flag=True, help="Ensure obs index is unique.", show_default=True
)
@click.option(
"--make-var-names-unique", default=True, is_flag=True, help="Ensure var index is unique.", show_default=True
)
def prepare(
data,
layout,
recipe,
output,
plotting,
sparse,
overwrite,
set_obs_names,
set_var_names,
make_obs_names_unique,
make_var_names_unique,
):
"""Preprocesses data for use with cellxgene.
This tool runs a series of scanpy routines for preparing a dataset
@@ -35,6 +61,7 @@ def prepare(data, layout, recipe, output, plotting, sparse, overwrite,
# collect slow imports here to make CLI startup more responsive
click.echo("[cellxgene] Starting CLI...")
import matplotlib
matplotlib.use("Agg")
import scanpy.api as sc
@@ -49,8 +76,10 @@ def prepare(data, layout, recipe, output, plotting, sparse, overwrite,
output = expanduser(output)
if not output:
click.echo("Warning: No file will be saved, to save the results of cellxgene prepare include "
"--output <filename> to save output to a new file")
click.echo(
"Warning: No file will be saved, to save the results of cellxgene prepare include "
"--output <filename> to save output to a new file"
)
if isfile(output) and not overwrite:
raise click.UsageError(f"Cannot overwrite existing file {output}, try using the flag --overwrite")
@@ -119,9 +148,11 @@ def prepare(data, layout, recipe, output, plotting, sparse, overwrite,
try:
sc.tl.louvain(adata)
except ModuleNotFoundError:
click.echo("\nWarning: louvain module is not installed, no clusters will be calculated. "
"To fix this please install cellxgene with the optional feature louvain enabled: "
"`pip install cellxgene[louvain]`")
click.echo(
"\nWarning: louvain module is not installed, no clusters will be calculated. "
"To fix this please install cellxgene with the optional feature louvain enabled: "
"`pip install cellxgene[louvain]`"
)
def run_layout(adata):
if len(unique(adata.obs["louvain"].values)) < 10:
@@ -142,11 +173,11 @@ def prepare(data, layout, recipe, output, plotting, sparse, overwrite,
def show_step(item):
names = {
"make_sparse": "Ensuring sparsity",
"run_recipe": f"Running preprocessing recipe \"{recipe}\"",
"run_recipe": f'Running preprocessing recipe "{recipe}"',
"run_pca": "Running PCA",
"run_neighbors": "Calculating neighbors",
"run_louvain": "Calculating clusters",
"run_layout": "Computing layout"
"run_layout": "Computing layout",
}
if item is not None:
return names[item.__name__]
server/requirements-dev.txt
+2 -1
View File
@@ -1,5 +1,6 @@
black
bumpversion>=0.5
pytest>=3.6.3
requests>=2.18.4
twine>=1.12.1
bumpversion>=0.5
-r requirements.txt
server/test/test_api.py
+59 -80
View File
@@ -9,15 +9,7 @@ LOCAL_URL = "http://127.0.0.1:5005/"
VERSION = "v0.2"
URL_BASE = f"{LOCAL_URL}api/{VERSION}/"
BAD_FILTER = {
"filter": {
"obs": {
"annotation_value": [
{"name": "xyz"},
],
}
}
}
BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
class EndPoints(unittest.TestCase):
@@ -133,7 +125,7 @@ class EndPoints(unittest.TestCase):
{"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
{"name": "n_counts", "min": 3000},
],
"index": [1, 99, [1000, 2000]]
"index": [1, 99, [1000, 2000]],
}
}
}
@@ -154,7 +146,7 @@ class EndPoints(unittest.TestCase):
{"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
{"name": "n_counts", "min": 3000},
],
"index": [1, 99, [1000, 2000]]
"index": [1, 99, [1000, 2000]],
}
}
}
@@ -170,23 +162,9 @@ class EndPoints(unittest.TestCase):
url = f"{URL_BASE}{endpoint}"
params = {
"mode": "topN",
"set1": {
"filter": {
"obs": {"annotation_value": [
{"name": "louvain", "values": ["NK cells"]}
]
}
}
},
"set2": {
"filter": {
"obs": {"annotation_value": [
{"name": "louvain", "values": ["CD8 T cells"]}
]
}
}
},
"count": 7
"set1": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["NK cells"]}]}}},
"set2": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["CD8 T cells"]}]}}},
"count": 7,
}
result = self.session.post(url, json=params)
self.assertEqual(result.status_code, HTTPStatus.OK)
@@ -199,20 +177,8 @@ class EndPoints(unittest.TestCase):
params = {
"mode": "topN",
"count": 10,
"set1": {
"filter": {
"obs": {
"index": [[0, 500]]
}
}
},
"set2": {
"filter": {
"obs": {
"index": [[500, 1000]]
}
}
}
"set1": {"filter": {"obs": {"index": [[0, 500]]}}},
"set2": {"filter": {"obs": {"index": [[500, 1000]]}}},
}
result = self.session.post(url, json=params)
self.assertEqual(result.status_code, HTTPStatus.OK)
@@ -249,15 +215,7 @@ class EndPoints(unittest.TestCase):
def test_put_annotations_var(self):
endpoint = "annotations/var"
url = f"{URL_BASE}{endpoint}"
var_filter = {
"filter": {
"var": {
"annotation_value": [
{"name": "name", "values": ["ATAD3C", "RER1"]},
]
}
}
}
var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}]}}}
result = self.session.put(url, json=var_filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
result_data = result.json()
@@ -268,15 +226,7 @@ class EndPoints(unittest.TestCase):
endpoint = "annotations/var"
query = "annotation-name=n_cells"
url = f"{URL_BASE}{endpoint}?{query}"
var_filter = {
"filter": {
"var": {
"annotation_value": [
{"name": "name", "values": ["ATAD3C", "RER1"]},
]
}
}
}
var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}]}}}
result = self.session.put(url, json=var_filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
result_data = result.json()
@@ -335,7 +285,7 @@ class EndPoints(unittest.TestCase):
{"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
{"name": "n_counts", "min": 3000},
],
"index": [1, 99, [1000, 2000]]
"index": [1, 99, [1000, 2000]],
}
}
}
@@ -349,15 +299,7 @@ class EndPoints(unittest.TestCase):
endpoint = f"data/{axis}"
url = f"{URL_BASE}{endpoint}"
header = {"Accept": "application/json"}
var_filter = {
"filter": {
"var": {
"annotation_value": [
{"name": "name", "values": ["RER1"]},
]
}
}
}
var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}}}
result = self.session.put(url, headers=header, json=var_filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
result_data = result.json()
@@ -371,16 +313,44 @@ class EndPoints(unittest.TestCase):
def test_cache(self):
endpoint = "annotations/var"
url = f"{URL_BASE}{endpoint}"
f1 = {"filter": {"var": {"annotation_value": [{"name": "name",
"values": ["HLA-DRB1", "HLA-DQA1", "HLA-DQB1", "HLA-DPA1",
"HLA-DPB1", "MS4A1", "IL32", "CCL5", "CD79B",
"CD79A"]}]}}}
f1 = {
"filter": {
"var": {
"annotation_value": [
{
"name": "name",
"values": [
"HLA-DRB1",
"HLA-DQA1",
"HLA-DQB1",
"HLA-DPA1",
"HLA-DPB1",
"MS4A1",
"IL32",
"CCL5",
"CD79B",
"CD79A",
],
}
]
}
}
}
result = self.session.put(url, json=f1)
self.assertEqual(result.status_code, HTTPStatus.OK)
result_data1 = result.json()
f2 = {"filter": {"var": {"annotation_value": [{"name": "name",
"values": ["FGFBP2", "GZMA", "LTB", "PRF1", "CTSW", "GZMH",
"CCL5", "CCL4", "CST7", "NKG7"]}]}}}
f2 = {
"filter": {
"var": {
"annotation_value": [
{
"name": "name",
"values": ["FGFBP2", "GZMA", "LTB", "PRF1", "CTSW", "GZMH", "CCL5", "CCL4", "CST7", "NKG7"],
}
]
}
}
}
result = self.session.put(url, json=f2)
self.assertEqual(result.status_code, HTTPStatus.OK)
result_data2 = result.json()
@@ -393,9 +363,18 @@ class EndPoints(unittest.TestCase):
result = self.session.put(url, json=f1)
self.assertEqual(result.status_code, HTTPStatus.OK)
result_data1 = result.json()
f2 = {"filter": {"var": {"annotation_value": [{"name": "name",
"values": ["FGFBP2", "GZMA", "LTB", "PRF1", "CTSW", "GZMH",
"CCL5", "CCL4", "CST7", "NKG7"]}]}}}
f2 = {
"filter": {
"var": {
"annotation_value": [
{
"name": "name",
"values": ["FGFBP2", "GZMA", "LTB", "PRF1", "CTSW", "GZMH", "CCL5", "CCL4", "CST7", "NKG7"],
}
]
}
}
}
result = self.session.put(url, json=f2)
self.assertEqual(result.status_code, HTTPStatus.OK)
result_data2 = result.json()
server/test/test_filter.py
+10 -8
View File
@@ -54,13 +54,17 @@ class UtilTest(unittest.TestCase):
def test_complex_filter(self):
filter_dict = ImmutableMultiDict(
[("obs:louvain", "NK cells"), ("obs:louvain", "CD8 T cells"), ("obs:n_counts", "3000,*")])
[("obs:louvain", "NK cells"), ("obs:louvain", "CD8 T cells"), ("obs:n_counts", "3000,*")]
)
filter_ = parse_filter(filter_dict, self.schema)
self.assertIn("obs", filter_)
self.assertEqual(filter_["obs"]["annotation_value"], [{"name": "louvain",
"values": ["NK cells", "CD8 T cells"]},
{"name": "n_counts",
"max": None, "min": 3000.0}])
self.assertEqual(
filter_["obs"]["annotation_value"],
[
{"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
{"name": "n_counts", "max": None, "min": 3000.0},
],
)
def test_bad_filter(self):
bad_annotation_type = ImmutableMultiDict([("obs:tissue", "lung")])
@@ -71,9 +75,7 @@ class UtilTest(unittest.TestCase):
parse_filter(bad_axis, self.schema)
def test_boolean_filter(self):
schema = {
"obs": [{"name": "bool_filter", "type": "boolean"}]
}
schema = {"obs": [{"name": "bool_filter", "type": "boolean"}]}
filter_dict = ImmutableMultiDict([("obs:bool_filter", "false")])
filter_ = parse_filter(filter_dict, schema)
self.assertIn("obs", filter_)
server/test/test_scanpy_engine.py
+30 -101
View File
@@ -3,7 +3,6 @@ from os import path
import pytest
import time
import unittest
import argparse
import numpy as np
from pandas import Series
@@ -13,9 +12,15 @@ from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
class UtilTest(unittest.TestCase):
def setUp(self):
args = {'layout': 'umap', 'diffexp': 'ttest', 'max_category_items': 100,
'obs_names': None, 'var_names': None, 'diffexp_lfc_cutoff': 0.01,
'nan_to_num': True}
args = {
"layout": "umap",
"diffexp": "ttest",
"max_category_items": 100,
"obs_names": None,
"var_names": None,
"diffexp_lfc_cutoff": 0.01,
"nan_to_num": True,
}
self.data = ScanpyEngine("example-dataset/pbmc3k.h5ad", args)
self.data._create_schema()
@@ -23,8 +28,8 @@ class UtilTest(unittest.TestCase):
def test_init(self):
self.assertEqual(self.data.cell_count, 2638)
self.assertEqual(self.data.gene_count, 1838)
epsilon = 0.000005
self.assertTrue(self.data.data.X[0, 0] - -0.17146951 < epsilon)
epsilon = 0.000_005
self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
def test_mandatory_annotations(self):
self.assertIn("name", self.data.data.obs)
@@ -39,69 +44,36 @@ class UtilTest(unittest.TestCase):
self.data._validate_data_types()
def test_filter_idx(self):
filter_ = {
"filter": {
"var": {
"index": [1, 99, [200, 300]]
},
"obs": {
"index": [1, 99, [1000, 2000]]
}
}
}
filter_ = {"filter": {"var": {"index": [1, 99, [200, 300]]}, "obs": {"index": [1, 99, [1000, 2000]]}}}
data = self.data.filter_dataframe(filter_["filter"])
self.assertEqual(data.shape, (1002, 102))
def test_filter_annotation(self):
filter_ = {
"filter": {
"obs": {
"annotation_value": [
{"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
]
}
}
"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["NK cells", "CD8 T cells"]}]}}
}
data = self.data.filter_dataframe(filter_["filter"])
self.assertEqual(data.shape, (470, 1838))
filter_ = {
"filter": {
"obs": {
"annotation_value": [
{"name": "n_counts", "min": 3000},
]
}
}
}
filter_ = {"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}}
data = self.data.filter_dataframe(filter_["filter"])
self.assertEqual(data.shape, (497, 1838))
def test_filter_annotation_no_uns(self):
filter_ = {
"filter": {
"var": {
"annotation_value": [
{"name": "name", "values": ["RER1"]},
]
}
}
}
filter_ = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}}}
data = self.data.filter_dataframe(filter_["filter"])
self.assertEqual(data.shape[1], 1)
def test_filter_complex(self):
filter_ = {
"filter": {
"var": {
"index": [1, 99, [200, 300]]
},
"var": {"index": [1, 99, [200, 300]]},
"obs": {
"annotation_value": [
{"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
{"name": "n_counts", "min": 3000},
],
"index": [1, 99, [1000, 2000]]
}
"index": [1, 99, [1000, 2000]],
},
}
}
data = self.data.filter_dataframe(filter_["filter"])
@@ -117,13 +89,14 @@ class UtilTest(unittest.TestCase):
self.assertEqual(self.data.schema, schema)
def test_schema_produces_error(self):
self.data.data.obs["time"] = Series(list([time.time() for i in range(self.data.cell_count)]),
dtype="datetime64[ns]")
self.data.data.obs["time"] = Series(
list([time.time() for i in range(self.data.cell_count)]), dtype="datetime64[ns]"
)
with pytest.raises(TypeError):
self.data._create_schema()
def test_config(self):
self.assertEqual(self.data.features["layout"]["obs"], {'available': True, 'interactiveLimit': 50000})
self.assertEqual(self.data.features["layout"]["obs"], {"available": True, "interactiveLimit": 50000})
def test_layout(self):
layout = self.data.layout(None)
@@ -153,16 +126,8 @@ class UtilTest(unittest.TestCase):
def test_filtered_annotation(self):
filter_ = {
"filter": {
"obs": {
"annotation_value": [
{"name": "n_counts", "min": 3000},
]
},
"var": {
"annotation_value": [
{"name": "name", "values": ["ATAD3C", "RER1"]},
]
}
"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]},
"var": {"annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}]},
}
}
annotations = self.data.annotation(filter_["filter"], "obs")
@@ -173,33 +138,13 @@ class UtilTest(unittest.TestCase):
self.assertEqual(len(annotations["data"]), 2)
def test_filtered_layout(self):
filter_ = {
"filter": {
"obs": {
"annotation_value": [
{"name": "n_counts", "min": 3000},
]
}
}
}
filter_ = {"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}}
layout = self.data.layout(filter_["filter"])
self.assertEqual(len(layout["coordinates"]), 497)
def test_diffexp_topN(self):
f1 = {
"filter": {
"obs": {
"index": [[0, 500]]
}
}
}
f2 = {
"filter": {
"obs": {
"index": [[500, 1000]]
}
}
}
f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
result = self.data.diffexp_topN(f1["filter"], f2["filter"])
self.assertEqual(len(result), 10)
result = self.data.diffexp_topN(f1["filter"], f2["filter"], 20)
@@ -214,15 +159,7 @@ class UtilTest(unittest.TestCase):
self.assertEqual(len(data_frame_var["obs"]), 2638)
def test_filtered_data_frame(self):
filter_ = {
"filter": {
"obs": {
"annotation_value": [
{"name": "n_counts", "min": 3000},
]
}
}
}
filter_ = {"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}}
data_frame_obs = self.data.data_frame(filter_["filter"], "obs")
self.assertEqual(len(data_frame_obs["var"]), 1838)
self.assertEqual(len(data_frame_obs["obs"]), 497)
@@ -236,15 +173,7 @@ class UtilTest(unittest.TestCase):
def test_data_single_gene(self):
for axis in ["obs", "var"]:
filter_ = {
"filter": {
"var": {
"annotation_value": [
{"name": "name", "values": ["RER1"]},
]
}
}
}
filter_ = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}}}
data_frame_var = self.data.data_frame(filter_["filter"], axis)
if axis == "obs":
self.assertEqual(type(data_frame_var["var"][0]), int)
@@ -253,5 +182,5 @@ class UtilTest(unittest.TestCase):
self.assertEqual(type(data_frame_var["obs"][0]), int)
self.assertIsInstance(data_frame_var["var"][0], (list, tuple))
if __name__ == '__main__':
if __name__ == "__main__":
unittest.main()
setup.cfg
+1
View File
@@ -1,2 +1,3 @@
[flake8]
max-line-length = 120
ignore = E203