Add backend to cellxgene repo

This commit is contained in:
Charlotte Weaver
2018-06-01 11:16:23 -07:00
parent 37df1d7a67
commit e84793ede9
98 changed files with 718 additions and 9 deletions
+33
View File
@@ -14,3 +14,36 @@ npm-debug.log
.vscode
data
*.idea*
__pycache__
*.DS_Store*
# Elastic Beanstalk Files
.elasticbeanstalk/*
!.elasticbeanstalk/*.cfg.yml
!.elasticbeanstalk/*.global.yml
GBM
venv
extesting
Dockerfile-*
*-data/*
data/*
runServer.py
templates/favicon.png
templates/index.html
templates/service-worker.js
templates/static/*
Graph.dot*
backend/app/web/static/css/
backend/app/web/static/img/
backend/app/web/static/js/
backend/app/web/templates/index\.html
+1 -9
View File
@@ -1,9 +1 @@
# cellxgene
A React + Redux web application for exploring large scale single cell RNA sequence data.
##### Quickstart:
* `npm install`
* `npm start`
* `localhost:3000`
cxg-v2
+63
View File
@@ -0,0 +1,63 @@
import os
from flask import Flask, url_for, g
from flask_compress import Compress
from flask_cors import CORS
from flask_restful_swagger_2 import get_swagger_blueprint
# from werkzeug.contrib.profiler import ProfilerMiddleware
from .web import webapp
from .rest_api.rest import get_api_resources
app = Flask(__name__)
Compress(app)
CORS(app)
#Config
CONFIG_FILE = os.environ.get("CXG_CONFIG_FILE", default="scanpy-test.cfg")
CXG_DIR = os.environ.get("CXG_DIRECTORY", default="/Users/charlotteweaver/Documents/Git/cxg-v2/")
SECRET_KEY = os.environ.get("CXG_SECRET_KEY", default="SparkleAndShine")
# TODO remove the 2 when this is prod
CXG_API_BASE = os.environ.get("CXG_API_BASE2", default="http://0.0.0.0:5005/api/")
if not CONFIG_FILE:
raise ValueError("No config file set for Flask application")
app.config.from_pyfile(os.path.join(CXG_DIR, "config", CONFIG_FILE), silent=True)
app.config.update(
SECRET_KEY=SECRET_KEY,
CXG_API_BASE=CXG_API_BASE,
)
app.config.update(
DATA=os.path.join(CXG_DIR, app.config["DATA_DIR"]),
)
app.config['PROFILE'] = True
# app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[15])
# Application Data
data = None
if app.config["ENGINE"] == "scanpy":
from .scanpy_engine.scanpy_engine import ScanpyEngine
data = ScanpyEngine(app.config["DATA"])
REACTIVE_LIMIT = 1_000_000
# A list of swagger document objects
docs = []
resources = get_api_resources()
docs.append(resources.get_swagger_doc())
app.register_blueprint(webapp.bp)
app.register_blueprint(resources.blueprint)
app.register_blueprint(get_swagger_blueprint(docs, '/api/swagger', produces=["application/json"], title="cellxgene rest api",
description='An API connecting ExpressionMatrix2 clustering algorithm to cellxgene'))
app.add_url_rule('/', endpoint='index')
if __name__ == "__main__":
app.run(host='0.0.0.0', debug=True, port=5005)
+57
View File
@@ -0,0 +1,57 @@
from abc import ABC, abstractmethod
class CXGDriver(metaclass=abc.ABCMeta):
def __init__(self, data, schema=None, graph_method=None, diffexp_method=None):
self.data = _load_data(data)
@abstractmethod
@staticmethod
def _load_data(data):
pass
@abstractmethod
def cells(self):
pass
@abstractmethod
def cellids(self):
pass
@abstractmethod
def genes(self):
pass
@abstractmethod
def filter_cells(self, filter):
"""
Filter cells from data and return a subset of the data
:param filter:
:return: iterator through cells
"""
pass
@abstractmethod
def metadata_ranges(self, cells_iterator):
"""
"""
pass
@abstractmethod
def metadata(self, cells_iterator):
"""
"""
pass
@abstractmethod
def create_graph(self, cells_iterator):
"""
"""
pass
@abstractmethod
def diffexp(self, cells_iterator):
"""
"""
pass
+215
View File
@@ -0,0 +1,215 @@
from flask import (
Blueprint, render_template, request, url_for
)
from flask_restful_swagger_2 import Api, swagger, Resource
from ..util.utils import make_payload
from ..util.filter import parse_filter, QueryStringError
class InitializeAPI(Resource):
@swagger.doc({
'summary': 'get metadata schema, ranges for values, and cell count to initialize cellxgene app',
'tags': ['initialize'],
'parameters': [],
'responses': {
'200': {
'description': 'initialization data for UI',
'examples': {
'application/json': {
"data": {
"cellcount": 3589,
"options": {
"Sample.type": {
"options": {
"Glioblastoma": 3589
}
},
"Selection": {
"options": {
"Astrocytes(HEPACAM)": 714,
"Endothelial(BSC)": 123,
"Microglia(CD45)": 1108,
"Neurons(Thy1)": 685,
"Oligodendrocytes(GC)": 294,
"Unpanned": 665
}
},
"Splice_sites_AT.AC": {
"range": {
"max": 1025,
"min": 152
}
},
"Splice_sites_Annotated": {
"range": {
"max": 1075869,
"min": 26
}
}
},
"schema": {
"CellName": {
"displayname": "Name",
"type": "string",
"variabletype": "categorical"
},
"Class": {
"displayname": "Class",
"type": "string",
"variabletype": "categorical"
},
"ERCC_reads": {
"displayname": "ERCC Reads",
"type": "int",
"variabletype": "continuous"
},
"ERCC_to_non_ERCC": {
"displayname": "ERCC:Non-ERCC",
"type": "float",
"variabletype": "continuous"
},
"Genes_detected": {
"displayname": "Genes Detected",
"type": "int",
"variabletype": "continuous"
}
},
"genes": ["1/2-SBSRNA4", "A1BG", "A1BG-AS1"]
},
"status": {
"error": False,
"errormessage": ""
}
}
}
}
}
})
def get(self):
from cxg import data, REACTIVE_LIMIT
return make_payload({
"schema": data.schema,
"ranges": data.metadata_ranges(data.ADATA, data.schema),
"cellcount": data.cell_count(),
"reactivelimit": REACTIVE_LIMIT,
"genes": data.all_genes(),
})
class CellsAPI(Resource):
@swagger.doc({
'summary': 'filter based on metadata fields to get a subset cells, expression data, and metadata',
'tags': ['cells'],
'description': "Cells takes query parameters defined in the schema retrieved from the /initialize enpoint. "
"<br>For categorical metadata keys filter based on `key=value` <br>"
" For continuous metadata keys filter by `key=min,max`<br> Either value "
"can be replaced by a \*. To have only a minimum value `key=min,\*` To have only a maximum "
"value `key=\*,max` <br>Graph data (if retrieved) is normalized"
" To only retrieve cells that don't have a value for the key filter by `key`",
'parameters': [],
'responses': {
'200': {
'description': 'initialization data for UI',
'examples': {
'application/json': {
"data": {
"badmetadatacount": 0,
"cellcount": 0,
"cellids": ["..."],
"metadata": [
{
"CellName": "1001000173.G8",
"Class": "Neoplastic",
"Cluster_2d": "11",
"Cluster_2d_color": "#8C564B",
"Cluster_CNV": "1",
"Cluster_CNV_color": "#1F77B4",
"ERCC_reads": "152104",
"ERCC_to_non_ERCC": "0.562454470489481",
"Genes_detected": "1962",
"Location": "Tumor",
"Location.color": "#FF7F0E",
"Multimapping_reads_percent": "2.67",
"Neoplastic": "Neoplastic",
"Non_ERCC_reads": "270429",
"Sample.name": "BT_S2",
"Sample.name.color": "#AEC7E8",
"Sample.type": "Glioblastoma",
"Sample.type.color": "#1F77B4",
"Selection": "Unpanned",
"Selection.color": "#98DF8A",
"Splice_sites_AT.AC": "102",
"Splice_sites_Annotated": "122397",
"Splice_sites_GC.AG": "761",
"Splice_sites_GT.AG": "125741",
"Splice_sites_non_canonical": "56",
"Splice_sites_total": "126660",
"Total_reads": "1741039",
"Unique_reads": "1400382",
"Unique_reads_percent": "80.43",
"Unmapped_mismatch": "2.15",
"Unmapped_other": "0.18",
"Unmapped_short": "14.56",
"housekeeping_cluster": "2",
"housekeeping_cluster_color": "#AEC7E8",
"recluster_myeloid": "NA",
"recluster_myeloid_color": "NA"
},
],
"reactive": True,
"graph": [
[
"1001000173.G8",
0.93836,
0.28623
],
[
"1001000173.D4",
0.1662,
0.79438
]
],
"status": {
"error": False,
"errormessage": ""
}
},
}
},
},
'400': {
'description': 'bad query params',
}
}
})
def get(self):
from cxg import data
payload = {
"cellids": [],
"metadata": [],
"cellcount": 0,
"graph": [],
"ranges": {},
}
# get query params
filter = parse_filter(request.args, data.schema)
filtered_data = data.filter_cells(filter)
payload["metadata"], payload["cellids"] = data.metadata(filtered_data)
payload["ranges"] = data.metadata_ranges(filtered_data, data.schema)
payload["cellcount"] = len(payload["cellids"])
payload["graph"] = data.create_graph(filtered_data)
return make_payload(payload)
def get_api_resources():
bp = Blueprint('api', __name__, url_prefix='/api/v2.0')
api = Api(bp, add_api_spec_resource=False)
api.add_resource(InitializeAPI, "/initialize")
api.add_resource(CellsAPI, "/cells")
return api
@@ -0,0 +1,86 @@
from os.path import join
import scanpy.api as sc
import numpy as np
from ..util.schema_parse import parse_schema
class ScanpyEngine():
def __init__(self, dataloc):
self.ADATA = sc.read(join(dataloc, "data.h5ad"))
self.cell_count = self._cell_count
self.schema = parse_schema(join(dataloc, "data_schema.json"))
def _cell_count(self):
return len(self.ADATA.obs.index)
def all_cells(self):
return self.ADATA.obs.index.tolist()
def all_genes(self):
return self.ADATA.var.index.tolist()
def gene_count(self):
return len(self.ADATA.var.index)
def filter_cells(self, filter):
cell_idx = np.ones((self.cell_count(),), dtype=bool)
for key, value in filter.items():
if value["variable_type"] == "categorical":
key_idx = np.in1d(getattr(self.ADATA.obs, key), value["query"])
cell_idx = np.logical_and(cell_idx, key_idx)
else:
min_ = value["query"]["min"]
max_ = value["query"]["max"]
if min_:
key_idx = np.array((getattr(self.ADATA.obs, key) >= min_).data)
cell_idx = np.logical_and(cell_idx, key_idx)
if max_:
key_idx = np.array((getattr(self.ADATA.obs, key) <= min_).data)
cell_idx = np.logical_and(cell_idx, key_idx)
return self.ADATA[cell_idx, :]
@staticmethod
def metadata_ranges(data, schema):
metadata_ranges = {}
for field in schema:
if schema[field]["variabletype"] == "categorical":
group_by = field
if group_by == "CellName":
group_by = 'index'
metadata_ranges[field] = {"options": data.obs.groupby(group_by).size().to_dict()}
else:
metadata_ranges[field] = {
"range": {
"min": data.obs[field].min(),
"max": data.obs[field].max()
}
}
return metadata_ranges
@staticmethod
def metadata(data):
cell_ids = []
metadata = data.obs.to_dict(orient="records")
# Do i have to loop twice?
for idx, cell_name in enumerate(data.obs.index):
metadata[idx]["CellName"] = cell_name
cell_ids.append(cell_name)
return metadata, cell_ids
@staticmethod
# TODO cache this
# TODO accept n-dim versions too
# TODO allow optional kw params to function
def create_graph(data, graph_method="umap"):
# Run the graph method
getattr(sc.tl, graph_method)(data)
graph = data.obsm["X_{graph_method}".format(graph_method=graph_method)]
normalized_graph = (graph - graph.min()) / (graph.max() - graph.min())
return np.hstack((data.obs.index.values.reshape(len(data.obs.index), 1), normalized_graph)).tolist()
View File
+62
View File
@@ -0,0 +1,62 @@
class QueryStringError(Exception):
pass
def _convert_variable(datatype, variable):
"""
Convert variable to number (float/int)
Used for dataset metadata and for query string
:param datatype: type to convert to
:param variable: value of variable
:return: converted variable
:raises: ValueError
"""
try:
if variable and datatype == "int":
variable = int(variable)
elif variable and datatype == "float":
variable = float(variable)
return variable
except ValueError:
raise
def parse_filter(filter, schema):
"""
{key: variable_type
value_type
query
:param filter:
:param schema:
:return:
"""
query = {}
for key in filter:
value = filter.getlist(key)
if key not in schema:
raise QueryStringError("Error: key {} not in metadata schema".format(key))
query[key] = {
"variable_type": schema[key]["variabletype"],
"value_type": schema[key]["type"]
}
if query[key]["variable_type"] == "categorical":
query[key]["query"] = _convert_variable(query[key]["value_type"], value)
elif query[key]["variable_type"] == "continuous":
value = value[0]
try:
min, max = value.split(",")
except ValueError:
raise QueryStringError("Error: min,max format required for range for key {}, got {}".format(key, value))
if min == "*":
min = None
if max == "*":
max = None
try:
query[key]["query"] = {
"min": _convert_variable(query[key]["value_type"], min),
"max": _convert_variable(query[key]["value_type"], max)
}
except ValueError:
raise QueryStringError(
"Error: expected type {} for key {}, got {}".format(query[key]["type"], key, value)
)
return query
+7
View File
@@ -0,0 +1,7 @@
import json
def parse_schema(filename):
with open(filename) as fh:
schema = json.load(fh)
return schema
+34
View File
@@ -0,0 +1,34 @@
import json
from numpy import float32, integer
from flask import make_response, jsonify
class Float32JSONEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, float32):
return float(obj)
elif isinstance(obj, integer):
return int(obj)
return json.JSONEncoder.default(self, obj)
def make_payload(data, errormessage="", errorcode=200):
"""
Creates JSON respons for requests
:param data: json data
:param errormessage: error message
:param errorcode: http error code
:return: flask json repsonse
"""
error = False
if errormessage:
error = True
# Questionable
data = json.loads(json.dumps(data, cls=Float32JSONEncoder))
return make_response(jsonify({
"data": data,
"status": {
"error": error,
"errormessage": errormessage,
}
}), errorcode)
View File
+97
View File
@@ -0,0 +1,97 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>CellxGene REST API - Swagger definition</title>
<link href="https://fonts.googleapis.com/css?family=Open+Sans:400,700|Source+Code+Pro:300,600|Titillium+Web:400,600,700"
rel="stylesheet">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.2.1/swagger-ui.css"
crossorigin="anonymous"/>
<style>
html {
box-sizing: border-box;
overflow: -moz-scrollbars-vertical;
overflow-y: scroll;
}
*,
*:before,
*:after {
box-sizing: inherit;
}
body {
margin: 0;
background: #fafafa;
}
</style>
</head>
<body>
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"
style="position:absolute;width:0;height:0">
<defs>
<symbol viewBox="0 0 20 20" id="unlocked">
<path d="M15.8 8H14V5.6C14 2.703 12.665 1 10 1 7.334 1 6 2.703 6 5.6V6h2v-.801C8 3.754 8.797 3 10 3c1.203 0 2 .754 2 2.199V8H4c-.553 0-1 .646-1 1.199V17c0 .549.428 1.139.951 1.307l1.197.387C5.672 18.861 6.55 19 7.1 19h5.8c.549 0 1.428-.139 1.951-.307l1.196-.387c.524-.167.953-.757.953-1.306V9.199C17 8.646 16.352 8 15.8 8z"></path>
</symbol>
<symbol viewBox="0 0 20 20" id="locked">
<path d="M15.8 8H14V5.6C14 2.703 12.665 1 10 1 7.334 1 6 2.703 6 5.6V8H4c-.553 0-1 .646-1 1.199V17c0 .549.428 1.139.951 1.307l1.197.387C5.672 18.861 6.55 19 7.1 19h5.8c.549 0 1.428-.139 1.951-.307l1.196-.387c.524-.167.953-.757.953-1.306V9.199C17 8.646 16.352 8 15.8 8zM12 8H8V5.199C8 3.754 8.797 3 10 3c1.203 0 2 .754 2 2.199V8z"/>
</symbol>
<symbol viewBox="0 0 20 20" id="close">
<path d="M14.348 14.849c-.469.469-1.229.469-1.697 0L10 11.819l-2.651 3.029c-.469.469-1.229.469-1.697 0-.469-.469-.469-1.229 0-1.697l2.758-3.15-2.759-3.152c-.469-.469-.469-1.228 0-1.697.469-.469 1.228-.469 1.697 0L10 8.183l2.651-3.031c.469-.469 1.228-.469 1.697 0 .469.469.469 1.229 0 1.697l-2.758 3.152 2.758 3.15c.469.469.469 1.229 0 1.698z"/>
</symbol>
<symbol viewBox="0 0 20 20" id="large-arrow">
<path d="M13.25 10L6.109 2.58c-.268-.27-.268-.707 0-.979.268-.27.701-.27.969 0l7.83 7.908c.268.271.268.709 0 .979l-7.83 7.908c-.268.271-.701.27-.969 0-.268-.269-.268-.707 0-.979L13.25 10z"/>
</symbol>
<symbol viewBox="0 0 20 20" id="large-arrow-down">
<path d="M17.418 6.109c.272-.268.709-.268.979 0s.271.701 0 .969l-7.908 7.83c-.27.268-.707.268-.979 0l-7.908-7.83c-.27-.268-.27-.701 0-.969.271-.268.709-.268.979 0L10 13.25l7.418-7.141z"/>
</symbol>
<symbol viewBox="0 0 24 24" id="jump-to">
<path d="M19 7v4H5.83l3.58-3.59L8 6l-6 6 6 6 1.41-1.41L5.83 13H21V7z"/>
</symbol>
<symbol viewBox="0 0 24 24" id="expand">
<path d="M10 18h4v-2h-4v2zM3 6v2h18V6H3zm3 7h12v-2H6v2z"/>
</symbol>
</defs>
</svg>
<div id="swagger-ui"></div>
<script src="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.2.1/swagger-ui-bundle.js"
crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.2.1/swagger-ui-standalone-preset.js"
crossorigin="anonymous"></script>
<script>
window.onload = function () {
const ui = SwaggerUIBundle({
url: window.location.origin + "/api/swagger.json",
dom_id: '#swagger-ui',
deepLinking: true,
presets: [
SwaggerUIBundle.presets.apis,
SwaggerUIStandalonePreset
],
plugins: [
SwaggerUIBundle.plugins.DownloadUrl
],
layout: "StandaloneLayout"
});
window.ui = ui
}
</script>
</body>
</html>
+22
View File
@@ -0,0 +1,22 @@
from flask import (
Blueprint, render_template, request, url_for, current_app
)
bp = Blueprint('webapp', __name__, template_folder='templates')
@bp.route('/')
def index():
url_base = current_app.config["CXG_API_BASE"]
dataset_title = current_app.config["DATASET_TITLE"]
return render_template("index.html", prefix=url_base, datasetTitle=dataset_title)
# renders swagger documentation
@bp.route('/swagger')
def swag():
return render_template("swagger.html")
# renders swagger documentation
@bp.route('/favicon.png')
def favicon():
return url_for("static", filename="img/favicon.png")
+3
View File
@@ -0,0 +1,3 @@
from cxg import app
app.run(host='0.0.0.0', debug=True, port=5005)
File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 43 KiB

After

Width:  |  Height:  |  Size: 43 KiB

View File
View File