Merge branch 'master' of https://github.com/chanzuckerberg/cellxgene into bkmartinjr-crossfiltergroups

.gitignore

@@ -14,3 +14,39 @@ npm-debug.log
 .vscode
 
 data
+
+
+*.idea*
+
+__pycache__
+*.DS_Store*
+
+# Elastic Beanstalk Files
+.elasticbeanstalk/*
+!.elasticbeanstalk/*.cfg.yml
+!.elasticbeanstalk/*.global.yml
+
+GBM
+venv
+extesting
+
+Dockerfile-*
+*-data/*
+data/*
+runServer.py
+templates/favicon.png
+templates/index.html
+templates/service-worker.js
+templates/static/*
+Graph.dot*
+
+server/app/web/static/css/
+server/app/web/static/img/
+server/app/web/static/js/
+server/app/web/templates/index\.html
+
+
+*.egg-info
+dist/*
+build/*
+

.travis.yml

@@ -0,0 +1,19 @@
+language: python
+python:
+  - "3.6"
+node_js:
+  - "8"
+cache:
+  pip: true
+install:
+  - set -eo pipefail
+  - pip install flake8 httpie
+  - ./bin/build-client
+  - python setup.py install
+script:
+  - set -eo pipefail
+  - flake8 server/app/
+  - pytest -s server/test/test_filter.py server/test/test_scanpy_engine.py
+  - cellxgene &
+  - for i in {1..90}; do if http :5005/api/v0.1/initialize > /dev/null; then break; else echo "Waiting for server..."; sleep 1; fi; done
+  - pytest server/test/test_api.py

MANIFEST.in

@@ -0,0 +1,3 @@
+recursive-include server/app/web/templates *
+recursive-include server/app/web/static *
+

README.md

@@ -2,8 +2,37 @@
 
 A React + Redux web application for exploring large scale single cell RNA sequence data.
 
-##### Quickstart:
+### Requirements
+- OS: OSX, Windows, Linux
+- python 3.6
+- npm
+- Google Chrome
 
-* `npm install`
+
-* `npm start`
+## Installation
-* `localhost:3000`
+
+#### clone project
+
+    git clone https://github.com/chanzuckerberg/cellxgene.git
+
+#### install client 
+
+    cd cellxgene
+    ./bin/build-client
+
+#### To use with virtual env for python (optional, but recommended)
+
+    ENV_NAME=cellxgene
+    python3 -m venv ${ENV_NAME}
+    source ${ENV_NAME}/bin/activate
+
+#### install server
+
+    python3 setup.py install
+
+#### run (with demo data)
+
+    cellxgene
+   
+   
+*Thanks to Alex Wolf his help with test data*

bin/build-client

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+npm install --prefix client/ client
+npm run  --prefix client build
+mkdir -p server/app/web/static/img
+cp client/build/index.html server/app/web/templates/
+cp -r client/build/static server/app/web/
+
+cp client/build/favicon.png server/app/web/static/img
+cp client/build/service-worker.js server/app/web/static/js/

client/index.html

@@ -3,7 +3,7 @@
   <head>
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1">
-    <title>Cellx</title>
+    <title>cellxgene</title>
     <link href="https://fonts.googleapis.com/css?family=Roboto:400,400i,700" rel="stylesheet">
     <style>
       html, body, p, h1, h2, h3, h4, h5, h6, span, button, input {

client/index_template.html

@@ -3,7 +3,7 @@
   <head>
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1">
-    <title>Cellx</title>
+    <title>cellxgene</title>
     <style>
       html, body, p, h1, h2, h3, h4, h5, h6, span, button, input {
         font-family: Helvetica Neue,Helvetica,Arial,sans-serif;

client/package.json

@@ -115,6 +115,8 @@
     "whatwg-fetch": "^2.0.1"
   },
   "jest": {
-    "testMatch": ["**/__tests__/**/?(*.)(spec|test).js?(x)"]
+    "testMatch": [
+      "**/__tests__/**/?(*.)(spec|test).js?(x)"
+    ]
   }
 }

client/src/components/continuous/histogramBrush.js

@@ -11,7 +11,6 @@ import FaPaintBrush from "react-icons/lib/fa/paint-brush";
 import * as globals from "../../globals";
 
 @connect(state => {
-  console.log("state in histo brush", state)
   const ranges =
     state.cells.cells && state.cells.cells.data.ranges
       ? state.cells.cells.data.ranges
@@ -172,7 +171,9 @@ class HistogramBrush extends React.Component {
     this.props.dispatch({
       type: "color by continuous metadata",
       colorAccessor: this.props.metadataField,
-      rangeMaxForColorAccessor: this.props.initializeRanges[this.props.metadataField].range.max
+      rangeMaxForColorAccessor: this.props.initializeRanges[
+        this.props.metadataField
+      ].range.max
     });
   }
   render() {

client/src/components/graph/drawPointsRegl.js

@@ -39,13 +39,7 @@ export default function(regl) {
       distance: regl.prop("distance"),
       view: regl.prop("view"),
       projection: (context, props) => {
-        return mat4.perspective(
+        return mat4.perspective([], Math.PI / 2, 1, 0.01, 1000);
-          [],
-          Math.PI / 2,
-          context.viewportWidth * props.scale / context.viewportHeight,
-          0.01,
-          1000
-        );
       }
     },
 

client/src/components/graph/graph.js

@@ -47,6 +47,39 @@ class Graph extends React.Component {
       mode: "brush"
     };
   }
+  reglDraw(regl, drawPoints, sizeBuffer, colorBuffer, pointBuffer, camera) {
+    regl.clear({
+      depth: 1,
+      color: [1, 1, 1, 1]
+    });
+    drawPoints({
+      size: sizeBuffer,
+      distance: camera.distance,
+      color: colorBuffer,
+      position: pointBuffer,
+      count: this.count,
+      view: camera.view()
+    });
+  }
+  restartReglLoop() {
+    const reglRender = this.state.regl.frame(() => {
+      this.reglDraw(
+        this.state.regl,
+        this.state.drawPoints,
+        this.state.sizeBuffer,
+        this.state.colorBuffer,
+        this.state.pointBuffer,
+        this.state.camera
+      );
+      this.state.camera.tick();
+    });
+
+    this.reglRenderState = "rendering";
+
+    this.setState({
+      reglRender
+    });
+  }
   componentDidMount() {
     // setup canvas and camera
     const camera = _camera(this.reglCanvas, { scale: true, rotate: false });
@@ -59,34 +92,31 @@ class Graph extends React.Component {
     const colorBuffer = regl.buffer();
     const sizeBuffer = regl.buffer();
 
-    regl.frame(({ viewportWidth, viewportHeight }) => {
+    /* first time, but this duplicates above function, should be possile to avoid this */
-      regl.clear({
+    const reglRender = regl.frame(() => {
-        depth: 1,
+      this.reglDraw(
-        color: [1, 1, 1, 1]
+        regl,
-      });
+        drawPoints,
-
+        sizeBuffer,
-      drawPoints({
+        colorBuffer,
-        size: sizeBuffer,
+        pointBuffer,
-        distance: camera.distance,
+        camera
-        color: colorBuffer,
+      );
-        position: pointBuffer,
-        count: this.count,
-        view: camera.view(),
-        scale: viewportHeight / viewportWidth
-      });
-
-      this.setState({ camera });
       camera.tick();
     });
 
+    this.reglRenderState = "rendering";
+
     this.setState({
       regl,
+      drawPoints,
       pointBuffer,
       colorBuffer,
-      sizeBuffer
+      sizeBuffer,
+      camera,
+      reglRender
     });
   }
-
   componentWillReceiveProps(nextProps) {
     if (this.state.regl && nextProps.crossfilter) {
       /* update the regl state */
@@ -156,6 +186,16 @@ class Graph extends React.Component {
       this.state.sizeBuffer({ data: this.renderCache.sizes, dimension: 1 });
 
       this.count = cellCount;
+
+      this.state.regl._refresh();
+      this.reglDraw(
+        this.state.regl,
+        this.state.drawPoints,
+        this.state.sizeBuffer,
+        this.state.colorBuffer,
+        this.state.pointBuffer,
+        this.state.camera
+      );
     }
 
     if (
@@ -164,8 +204,9 @@ class Graph extends React.Component {
       nextProps.responsive.width !== this.props.responsive.width
     ) {
       /* clear out whatever was on the div, even if nothing, but usually the brushes etc */
-      d3.select("#graphAttachPoint")
+      d3
-        .selectAll("*")
+        .select("#graphAttachPoint")
+        .selectAll("svg")
         .remove();
       const { svg, brush, brushContainer } = setupSVGandBrushElements(
         this.handleBrushSelectAction.bind(this),
@@ -176,6 +217,16 @@ class Graph extends React.Component {
       this.setState({ svg, brush, brushContainer });
     }
   }
+  componentDidUpdate() {
+    if (
+      this.state.reglRender &&
+      this.reglRenderState === "rendering" &&
+      this.state.mode !== "zoom"
+    ) {
+      this.state.reglRender.cancel();
+      this.reglRenderState = "paused";
+    }
+  }
   handleBrushSelectAction() {
     /* This conditional handles procedural brush deselect. Brush emits an event on procedural deselect because it is move: null */
     if (d3.event.sourceEvent !== null) {
@@ -200,7 +251,7 @@ class Graph extends React.Component {
       // transform screen coordinates -> cell coordinates
       const invert = pin => {
         const x =
-          (2 * pin[0]) / (this.props.responsive.height - this.graphPaddingTop) -
+          2 * pin[0] / (this.props.responsive.height - this.graphPaddingTop) -
           1;
         const y =
           2 *
@@ -318,6 +369,7 @@ class Graph extends React.Component {
                 <button
                   onClick={() => {
                     this.handleBrushDeselectAction();
+                    this.restartReglLoop();
                     this.setState({ mode: "zoom" });
                   }}
                   style={{

client/src/components/leftsidebar.js

@@ -34,7 +34,7 @@ class LeftSideBar extends React.Component {
             width: "100%"
           }}
         >
-          CELLxGENE {globals.datasetTitle}{" "}
+          cellxgene {globals.datasetTitle}{" "}
         </p>
         <div style={{ padding: 10 }}>
           <button

client/src/reducers/controls.js

@@ -2,7 +2,7 @@
 import _ from "lodash";
 import { parseRGB } from "../util/parseRGB";
 import { createSchemaByDataSniffing } from "../util/schema";
-var crossfilter = require("../util/typedCrossfilter");
+import crossfilter from "../util/typedCrossfilter";
 
 // Deduce the correct crossfilter dimension type from a metadata
 // schema description.

client/src/util/typedCrossfilter/bitArray.js

@@ -251,4 +251,4 @@ class BitArray {
   }
 }
 
-module.exports = BitArray;
+export default BitArray;

client/src/util/typedCrossfilter/index.js

@@ -29,9 +29,9 @@ more complex API.  In a few cases, elements of that API were incorporated.
 
 */
 
-var PositiveIntervals = require("./positiveIntervals");
+import PositiveIntervals from "./positiveIntervals";
-var BitArray = require("./bitArray");
+import BitArray from "./bitArray";
-var Util = require("./util");
+import {fillRange, lowerBound, lowerBoundIndirect, upperBound, upperBoundIndirect} from "./util";
 
 class TypedCrossfilter {
   constructor(data) {
@@ -118,7 +118,7 @@ class ScalarDimension {
     this.value = array;
 
     // create sort index
-    this.index = Util.fillRange(new Uint32Array(this.crossfilter.data.length));
+    this.index = fillRange(new Uint32Array(this.crossfilter.data.length));
     this.index.sort((a, b) => array[a] - array[b]);
 
     // groups, if any
@@ -199,14 +199,14 @@ class ScalarDimension {
   // filter by value - exact match
   filterExact(value) {
     const newFilter = [
-      Util.lowerBoundIndirect(
+      lowerBoundIndirect(
         this.value,
         this.index,
         value,
         0,
         this.value.length
       ),
-      Util.upperBoundIndirect(
+      upperBoundIndirect(
         this.value,
         this.index,
         value,
@@ -227,14 +227,14 @@ class ScalarDimension {
     const newFilter = [];
     for (let v = 0, len = values.length; v < len; v++) {
       const intv = [
-        Util.lowerBoundIndirect(
+        lowerBoundIndirect(
           this.value,
           this.index,
           values[v],
           0,
           this.value.length
         ),
-        Util.upperBoundIndirect(
+        upperBoundIndirect(
           this.value,
           this.index,
           values[v],
@@ -253,14 +253,14 @@ class ScalarDimension {
   filterRange(range) {
     const newFilter = [];
     const intv = [
-      Util.lowerBoundIndirect(
+      lowerBoundIndirect(
         this.value,
         this.index,
         range[0],
         0,
         this.value.length
       ),
-      Util.upperBoundIndirect(
+      upperBoundIndirect(
         this.value,
         this.index,
         range[1],
@@ -380,7 +380,7 @@ class EnumDimension extends ScalarDimension {
     const enumLen = this.enumIndex.length;
     for (let i = 0; i < len; i++) {
       const v = value(data[i]);
-      const e = Util.lowerBound(this.enumIndex, v, 0, enumLen);
+      const e = lowerBound(this.enumIndex, v, 0, enumLen);
       array[i] = e;
     }
     return array;
@@ -388,14 +388,14 @@ class EnumDimension extends ScalarDimension {
 
   filterExact(value) {
     return super.filterExact(
-      Util.lowerBound(this.enumIndex, value, 0, this.enumIndex.length)
+      lowerBound(this.enumIndex, value, 0, this.enumIndex.length)
     );
   }
 
   filterEnum(values) {
     return super.filterEnum(
       values.map(v =>
-        Util.lowerBound(this.enumIndex, v, 0, this.enumIndex.length)
+        lowerBound(this.enumIndex, v, 0, this.enumIndex.length)
       )
     );
   }
@@ -403,7 +403,7 @@ class EnumDimension extends ScalarDimension {
   filterRange(range) {
     return super.filterEnum(
       range.map(v =>
-        Util.lowerBound(this.enumIndex, v, 0, this.enumIndex.length)
+        lowerBound(this.enumIndex, v, 0, this.enumIndex.length)
       )
     );
   }
@@ -609,4 +609,4 @@ crossfilter.TypedCrossfilter = TypedCrossfilter;
 crossfilter.ScalarDimension = ScalarDimension;
 crossfilter.EnumDimension = EnumDimension;
 
-module.exports = crossfilter;
+export default crossfilter;

client/src/util/typedCrossfilter/positiveIntervals.js

@@ -129,4 +129,4 @@ class PositiveIntervals {
   }
 }
 
-module.exports = PositiveIntervals;
+export default PositiveIntervals;

client/src/util/typedCrossfilter/util.js

@@ -8,7 +8,7 @@
 // fill an array or typedarray with a sequential range of numbers,
 // starting with `start`
 //
-function fillRange(arr, start = 0) {
+export function fillRange(arr, start = 0) {
   for (let i = 0, len = arr.length; i < len; i++) {
     arr[i] = i + start;
   }
@@ -30,7 +30,7 @@ function fillRange(arr, start = 0) {
 // a factory version of lowerBound that takes an accessor (rather than having
 // a special-cased version for lining the indirection).
 //
-function lowerBound(valueArray, value, first, last) {
+export function lowerBound(valueArray, value, first, last) {
   // this is just a binary search
   while (first < last) {
     const middle = (first + last) >>> 1;
@@ -45,7 +45,7 @@ function lowerBound(valueArray, value, first, last) {
 
 // Inlined performance optimization - used to indirect through a sort map.
 //
-function lowerBoundIndirect(valueArray, indexArray, value, first, last) {
+export function lowerBoundIndirect(valueArray, indexArray, value, first, last) {
   // this is just a binary search
   while (first < last) {
     const middle = (first + last) >>> 1;
@@ -69,7 +69,7 @@ function lowerBoundIndirect(valueArray, indexArray, value, first, last) {
 //    C++: upper_bound()
 //    Python: bisect.bisect_right()
 //
-function upperBound(valueArray, value, first, last) {
+export function upperBound(valueArray, value, first, last) {
   // this is just a binary search
   while (first < last) {
     const middle = (first + last) >>> 1;
@@ -84,7 +84,7 @@ function upperBound(valueArray, value, first, last) {
 
 // Inline performance optimization
 //
-function upperBoundIndirect(valueArray, indexArray, value, first, last) {
+export function upperBoundIndirect(valueArray, indexArray, value, first, last) {
   // this is just a binary search
   while (first < last) {
     const middle = (first + last) >>> 1;
@@ -96,11 +96,3 @@ function upperBoundIndirect(valueArray, indexArray, value, first, last) {
   }
   return first;
 }
-
-module.exports = {
-  fillRange,
-  lowerBound,
-  lowerBoundIndirect,
-  upperBound,
-  upperBoundIndirect
-};

example-dataset/data_schema.json

@@ -0,0 +1,32 @@
+{
+	"CellName": {
+		"type": "string",
+		"variabletype": "categorical",
+		"displayname": "Name",
+		"include": true
+	},
+	"n_genes": {
+		"type": "int",
+		"variabletype": "continuous",
+		"displayname": "Num Genes",
+		"include": true
+	},
+	"percent_mito": {
+		"type": "float",
+		"variabletype": "continuous",
+		"displayname": "Mitochondrial Percentage",
+		"include": true
+	},
+	"n_counts": {
+		"type": "float",
+		"variabletype": "continuous",
+		"displayname": "Num Counts",
+		"include": true
+	},
+    "louvain": {
+		"type": "string",
+		"variabletype": "categorical",
+		"displayname": "Louvain Cluster",
+		"include": true
+	}
+}

server/app/app.py

@@ -0,0 +1,57 @@
+import os
+
+from flask import Flask
+from flask_compress import Compress
+from flask_cors import CORS
+from flask_restful_swagger_2 import get_swagger_blueprint
+
+from .web import webapp
+from .rest_api.rest import get_api_resources
+
+REACTIVE_LIMIT = 1_000_000
+
+app = Flask(__name__)
+Compress(app)
+CORS(app)
+
+# Config
+CXG_DIR = os.environ.get("CXG_DIRECTORY", default="example-dataset/")
+SECRET_KEY = os.environ.get("CXG_SECRET_KEY", default="SparkleAndShine")
+ENGINE = os.environ.get("CXG_ENGINE", default="scanpy")
+TITLE = os.environ.get("DATASET_TITLE", default="PBMC 3K")
+# TODO remove the 2 when this is prod
+CXG_API_BASE = os.environ.get("CXG_API_BASE2", default="http://0.0.0.0:5005/api/")
+
+app.config.update(
+    SECRET_KEY=SECRET_KEY,
+    CXG_API_BASE=CXG_API_BASE,
+    ENGINE=ENGINE,
+    DATA=CXG_DIR,
+    DATASET_TITLE=TITLE
+)
+
+app.config["PROFILE"] = True
+# app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[15])
+
+# Application Data
+data = None
+if app.config["ENGINE"] == "scanpy":
+    from .scanpy_engine.scanpy_engine import ScanpyEngine
+    data = ScanpyEngine(app.config["DATA"], schema="data_schema.json")
+
+# A list of swagger document objects
+docs = []
+resources = get_api_resources()
+docs.append(resources.get_swagger_doc())
+
+app.register_blueprint(webapp.bp)
+app.register_blueprint(resources.blueprint)
+app.register_blueprint(
+    get_swagger_blueprint(docs, "/api/swagger", produces=["application/json"], title="cellxgene rest api",
+                          description="An API connecting ExpressionMatrix2 clustering algorithm to cellxgene"))
+
+app.add_url_rule("/", endpoint="index")
+
+
+def main():
+    app.run(host="0.0.0.0", debug=True, port=5005)

server/app/driver/driver.py

@@ -0,0 +1,81 @@
+from abc import ABCMeta, abstractmethod
+
+
+class CXGDriver(metaclass=ABCMeta):
+    def __init__(self, data, schema=None, graph_method=None, diffexp_method=None):
+        self.data = self._load_data(data)
+
+    @staticmethod
+    @abstractmethod
+    def _load_data(data):
+        pass
+
+    @abstractmethod
+    def _load_or_infer_schema(data):
+        pass
+
+    @abstractmethod
+    def cells(self):
+        pass
+
+    @abstractmethod
+    def genes(self):
+        pass
+
+    @abstractmethod
+    def filter_cells(self, filter):
+        """
+        Filter cells from data and return a subset of the data
+        A filter is a dictionary where the key is a metadatata category
+        Value is dictionary
+            value_type: int, float, string
+            variable_type: continuous, categorical
+            query: filter value, for categorical [val1, val2], for continuous {min: x, max:y}
+        Filters are combined with the and operator
+        :param filter:
+        :return: filtered dataframe
+        """
+        pass
+
+    @abstractmethod
+    def metadata(self, df, fields=None):
+        """
+         Gets metadata key:value for each cells
+
+        :param df: from filter_cells, dataframe
+        :param fields: list of keys for metadata to return, returns all metadata values if not set.
+        :return: list of metadata values
+        """
+        pass
+
+    @abstractmethod
+    def create_graph(self, df):
+        """
+        Computes a n-d layout for cells through dimensionality reduction.
+        :param df: from filter_cells, dataframe
+        :return:  [cellid, x, y]
+        """
+        pass
+
+    @abstractmethod
+    def diffexp(self, df1, df2):
+        """
+        Computes the top differentially expressed genes between two clusters
+        :param df1: from filter_cells, dataframe containing first set of cells
+        :param df2: from filter_cells, dataframe containing second set of cells
+        :return: top genes, stats and expression values for top genes
+        """
+        pass
+
+    @abstractmethod
+    def expression(self, df):
+        """
+        Retrieves expression for each gene for cells in data frame
+        :param df:
+        :return: {
+            "genes": list of genes,
+            "cells": list of cells and expression list,
+            "nonzero_gene_count": number of nonzero genes
+        }
+        """
+        pass

server/app/rest_api/rest.py

@@ -0,0 +1,439 @@
+from flask import (
+    Blueprint, request
+)
+from flask_restful_swagger_2 import Api, swagger, Resource
+
+from ..util.utils import make_payload
+from ..util.filter import parse_filter
+
+
+class InitializeAPI(Resource):
+    @swagger.doc({
+        "summary": "get metadata schema, ranges for values, and cell count to initialize cellxgene app",
+        "tags": ["initialize"],
+        "parameters": [],
+        "responses": {
+            "200": {
+                "description": "initialization data for UI",
+                "examples": {
+                    "application/json": {
+                        "data": {
+                            "cellcount": 3589,
+                            "options": {
+                                "Sample.type": {
+                                    "options": {
+                                        "Glioblastoma": 3589
+                                    }
+                                },
+                                "Selection": {
+                                    "options": {
+                                        "Astrocytes(HEPACAM)": 714,
+                                        "Endothelial(BSC)": 123,
+                                        "Microglia(CD45)": 1108,
+                                        "Neurons(Thy1)": 685,
+                                        "Oligodendrocytes(GC)": 294,
+                                        "Unpanned": 665
+                                    }
+                                },
+                                "Splice_sites_AT.AC": {
+                                    "range": {
+                                        "max": 1025,
+                                        "min": 152
+                                    }
+                                },
+                                "Splice_sites_Annotated": {
+                                    "range": {
+                                        "max": 1075869,
+                                        "min": 26
+                                    }
+                                }
+                            },
+                            "schema": {
+                                "CellName": {
+                                    "displayname": "Name",
+                                    "type": "string",
+                                    "variabletype": "categorical"
+                                },
+                                "Class": {
+                                    "displayname": "Class",
+                                    "type": "string",
+                                    "variabletype": "categorical"
+                                },
+                                "ERCC_reads": {
+                                    "displayname": "ERCC Reads",
+                                    "type": "int",
+                                    "variabletype": "continuous"
+                                },
+                                "ERCC_to_non_ERCC": {
+                                    "displayname": "ERCC:Non-ERCC",
+                                    "type": "float",
+                                    "variabletype": "continuous"
+                                },
+                                "Genes_detected": {
+                                    "displayname": "Genes Detected",
+                                    "type": "int",
+                                    "variabletype": "continuous"
+                                }
+                            },
+                            "genes": ["1/2-SBSRNA4", "A1BG", "A1BG-AS1"]
+
+                        },
+                        "status": {
+                            "error": False,
+                            "errormessage": ""
+                        }
+                    }
+                }
+            }
+        }
+    })
+    def get(self):
+        from server.app.app import data, REACTIVE_LIMIT
+        return make_payload({
+            "schema": data.schema,
+            "cellcount": data.cell_count,
+            "reactivelimit": REACTIVE_LIMIT,
+            "genes": data.genes(),
+            "ranges": data.metadata_ranges(),
+
+        })
+
+
+class CellsAPI(Resource):
+    @swagger.doc({
+        "summary": "filter based on metadata fields to get a subset cells, expression data, and metadata",
+        "tags": ["cells"],
+        "description": "Cells takes query parameters defined in the schema retrieved from the /initialize enpoint. "
+                       "<br>For categorical metadata keys filter based on `key=value` <br>"
+                       " For continuous metadata keys filter by `key=min,max`<br> Either value "
+                       "can be replaced by a \*. To have only a minimum value `key=min,\*`  To have only a maximum "
+                       "value `key=\*,max` <br>Graph data (if retrieved) is normalized"
+                       " To only retrieve cells that don't have a value for the key filter by `key`",
+        "parameters": [],
+
+        "responses": {
+            "200": {
+                "description": "initialization data for UI",
+                "examples": {
+                    "application/json": {
+                        "data": {
+                            "badmetadatacount": 0,
+                            "cellcount": 0,
+                            "cellids": ["..."],
+                            "metadata": [
+                                {
+                                    "CellName": "1001000173.G8",
+                                    "Class": "Neoplastic",
+                                    "Cluster_2d": "11",
+                                    "Cluster_2d_color": "#8C564B",
+                                    "Cluster_CNV": "1",
+                                    "Cluster_CNV_color": "#1F77B4",
+                                    "ERCC_reads": "152104",
+                                    "ERCC_to_non_ERCC": "0.562454470489481",
+                                    "Genes_detected": "1962",
+                                    "Location": "Tumor",
+                                    "Location.color": "#FF7F0E",
+                                    "Multimapping_reads_percent": "2.67",
+                                    "Neoplastic": "Neoplastic",
+                                    "Non_ERCC_reads": "270429",
+                                    "Sample.name": "BT_S2",
+                                    "Sample.name.color": "#AEC7E8",
+                                    "Sample.type": "Glioblastoma",
+                                    "Sample.type.color": "#1F77B4",
+                                    "Selection": "Unpanned",
+                                    "Selection.color": "#98DF8A",
+                                    "Splice_sites_AT.AC": "102",
+                                    "Splice_sites_Annotated": "122397",
+                                    "Splice_sites_GC.AG": "761",
+                                    "Splice_sites_GT.AG": "125741",
+                                    "Splice_sites_non_canonical": "56",
+                                    "Splice_sites_total": "126660",
+                                    "Total_reads": "1741039",
+                                    "Unique_reads": "1400382",
+                                    "Unique_reads_percent": "80.43",
+                                    "Unmapped_mismatch": "2.15",
+                                    "Unmapped_other": "0.18",
+                                    "Unmapped_short": "14.56",
+                                    "housekeeping_cluster": "2",
+                                    "housekeeping_cluster_color": "#AEC7E8",
+                                    "recluster_myeloid": "NA",
+                                    "recluster_myeloid_color": "NA"
+                                },
+                            ],
+                            "reactive": True,
+                            "graph": [
+                                [
+                                    "1001000173.G8",
+                                    0.93836,
+                                    0.28623
+                                ],
+
+                                [
+                                    "1001000173.D4",
+                                    0.1662,
+                                    0.79438
+                                ]
+
+                            ],
+                            "status": {
+                                "error": False,
+                                "errormessage": ""
+                            }
+
+                        },
+                    }
+                },
+            },
+
+            "400": {
+                "description": "bad query params",
+            }
+        }
+    })
+    def get(self):
+        from server.app.app import data
+        payload = {
+            "metadata": [],
+            "cellcount": 0,
+            "graph": [],
+            "ranges": {},
+        }
+        # get query params
+        cells_filter = parse_filter(request.args, data.schema)
+        filtered_data = data.filter_cells(cells_filter)
+        payload["metadata"] = data.metadata(filtered_data)
+        payload["ranges"] = data.metadata_ranges(filtered_data)
+        payload["graph"] = data.create_graph(filtered_data)
+        payload["cellcount"] = data.cell_count
+        return make_payload(payload)
+
+
+class ExpressionAPI(Resource):
+    @swagger.doc({
+        "summary": "Json with gene list and expression data by cell, limited to first 40 cells",
+        "tags": ["expression"],
+        "parameters": [
+            {
+                "name": "include_unexpressed_genes",
+                "description": "Include genes that have 0 expression across all cells in set",
+                "in": "path",
+                "type": "bool",
+            }
+        ],
+        "responses": {
+            "200": {
+                "description": "Json for heatmap",
+                "examples": {
+                    "application/json": {
+                        "data": {
+                            "cells": [
+                                {
+                                    "cellname": "1/2-SBSRNA4",
+                                    "e": [0, 0, 214, 0, 0]
+                                },
+                            ],
+                            "genes": [
+                                "1001000173.G8",
+                                "1001000173.D4",
+                                "1001000173.B4",
+                                "1001000173.A2",
+                                "1001000173.E2"
+                            ],
+                            "nonzero_gene_count": 2857
+                        },
+                        "status": {
+                            "error": False,
+                            "errormessage": ""
+                        }
+                    }
+                }
+            }
+        }
+    })
+    def get(self):
+        from server.app.app import data
+        expression_data = data.expression()
+        return make_payload(expression_data)
+
+    @swagger.doc({
+        "summary": "Json with gene list and expression data by cell",
+        "tags": ["expression"],
+        "parameters": [
+            {
+                "name": "body",
+                "in": "body",
+                "schema": {
+                    "example": {
+                        "celllist": ["1001000173.G8", "1001000173.D4"],
+                        "genelist": ["1/2-SBSRNA4", "A1BG", "A1BG-AS1", "A1CF", "A2LD1", "A2M", "A2ML1", "A2MP1",
+                                     "A4GALT"],
+                        "include_unexpressed_genes": True,
+                    }
+
+                }
+            },
+        ],
+        "responses": {
+            "200": {
+                "description": "Json for expressiondata",
+                "examples": {
+                    "application/json": {
+                        "data": {
+                            "cells": [
+                                {
+                                    "cellname": "1001000173.D4",
+                                    "e": [0, 0]
+                                },
+                                {
+                                    "cellname": "1001000173.G8",
+                                    "e": [0, 0]
+                                }
+                            ],
+                            "genes": [
+                                "ABCD4",
+                                "ZWINT"
+                            ],
+                            "nonzero_gene_count": 2857
+                        },
+                        "status": {
+                            "error": False,
+                            "errormessage": ""
+                        }
+
+                    }
+                }
+            },
+            "400": {
+                "description": "Required parameter missing/incorrect",
+            }
+        }
+    })
+    def post(self):
+        from server.app.app import data
+        args = request.get_json()
+        cell_list = args.get("celllist", [])
+        gene_list = args.get("genelist", [])
+        if not cell_list and not gene_list:
+            return make_payload([], "must include celllist and/or genelist parameter", 400)
+
+        expression_data = data.expression(cell_list, gene_list)
+
+        if cell_list and len(expression_data["cells"]) < len(cell_list):
+            return make_payload([], "Some cell ids not available", 400)
+        if gene_list and len(expression_data["genes"]) < len(gene_list):
+            return make_payload([], "Some genes not available", 400)
+
+        return make_payload(expression_data)
+
+
+class DifferentialExpressionAPI(Resource):
+    @swagger.doc({
+        "summary": "Get the top expressed genes for two cell sets. Calculated using t-test",
+        "tags": ["expression"],
+        "parameters": [
+            {
+                "name": "body",
+                "in": "body",
+                "schema": {
+                    "example": {
+                        "celllist1": ["1001000176.C12", "1001000176.C7", "1001000177.F11"],
+                        "celllist2": ["1001000012.D2", "1001000017.F10", "1001000033.C3", "1001000229.D4"],
+                        "num_genes": 5,
+                        "pval": 0.000001,
+                    },
+                }
+            }
+        ],
+        "responses": {
+            "200": {
+                "description": "top expressed genes for cellset1, cellset2",
+                "examples": {
+                    "application/json": {
+                        "data": {
+                            "celllist1": {
+                                "ave_diff": [
+                                    432.0132935431362,
+                                    12470.5623982637,
+                                    957.0246880086814
+                                ],
+                                "mean_expression_cellset1": [
+                                    438.6185567010309,
+                                    13315.536082474227,
+                                    1076.5773195876288
+                                ],
+                                "mean_expression_cellset2": [
+                                    6.605263157894737,
+                                    844.9736842105264,
+                                    119.55263157894737
+                                ],
+                                "pval": [
+                                    3.8906598089944563e-35,
+                                    1.9086226376018916e-25,
+                                    7.847480544069826e-21
+                                ],
+                                "topgenes": [
+                                    "TMSB10",
+                                    "FTL",
+                                    "TMSB4X"
+                                ]
+                            },
+                            "celllist2": {
+                                "ave_diff": [
+                                    -6860.599158979924,
+                                    -519.1314432989691,
+                                    -10278.328269126423
+                                ],
+                                "mean_expression_cellset1": [
+                                    2.8350515463917527,
+                                    0.6185567010309279,
+                                    23.09278350515464
+                                ],
+                                "mean_expression_cellset2": [
+                                    6863.434210526316,
+                                    519.75,
+                                    10301.421052631578
+                                ],
+                                "pval": [
+                                    4.662891833748732e-44,
+                                    3.6278087029927103e-37,
+                                    8.396825170618402e-35
+                                ],
+                                "topgenes": [
+                                    "SPARCL1",
+                                    "C1orf61",
+                                    "CLU"
+                                ]
+                            }
+                        },
+                        "status": {
+                            "error": False,
+                            "errormessage": ""
+                        }
+                    }
+                }
+            }
+        }
+    })
+    def post(self):
+        from server.app.app import data
+        args = request.get_json()
+        cell_list_1 = args.get("celllist1", [])
+        cell_list_2 = args.get("celllist2", [])
+        num_genes = args.get("num_genes", 7)
+        pval = args.get("pval", 0.5)
+        if not (cell_list_1 and cell_list_2):
+            return make_payload([],
+                                "must include celllist1 and celllist2 parameters",
+                                400)
+        data = data.diffexp(cell_list_1, cell_list_2, pval, num_genes)
+        return make_payload(data)
+
+
+def get_api_resources():
+    bp = Blueprint("api", __name__, url_prefix="/api/v0.1")
+    api = Api(bp, add_api_spec_resource=False)
+    api.add_resource(InitializeAPI, "/initialize")
+    api.add_resource(CellsAPI, "/cells")
+    api.add_resource(ExpressionAPI, "/expression")
+    api.add_resource(DifferentialExpressionAPI, "/diffexpression")
+    return api

server/app/scanpy_engine/scanpy_engine.py

@@ -0,0 +1,198 @@
+import os
+
+import numpy as np
+import scanpy.api as sc
+from scipy import stats
+
+from ..util.schema_parse import parse_schema
+from ..driver.driver import CXGDriver
+
+
+class ScanpyEngine(CXGDriver):
+
+    def __init__(self, data, schema=None, graph_method="umap", diffexp_method="ttest"):
+        self.data = self._load_data(data)
+        self.schema = self._load_or_infer_schema(data, schema)
+        self._set_cell_names()
+        self.cell_count = self.data.shape[0]
+        self.gene_count = self.data.shape[1]
+        self.graph_method = graph_method
+        self.diffexp_method = diffexp_method
+
+    def _set_cell_names(self):
+        self.data.obs["cell_name"] = list(self.data.obs.index)
+
+    @staticmethod
+    def _load_data(data):
+        return sc.read(os.path.join(data, "data.h5ad"))
+
+    @staticmethod
+    def _load_or_infer_schema(data, schema):
+        data_schema = None
+        if not schema:
+            pass
+        else:
+            data_schema = parse_schema(os.path.join(data, schema))
+        return data_schema
+
+    def cells(self):
+        return list(self.data.obs.index)
+
+    def genes(self):
+        return self.data.var.index.tolist()
+
+    def filter_cells(self, filter):
+        """
+        Filter cells from data and return a subset of the data
+        A filter is a dictionary where the key is a metadatata category
+        Value is dictionary
+            value_type: int, float, string
+            variable_type: continuous, categorical
+            query: filter value, for categorical [val1, val2], for continuous {min: x, max:y}
+        Filters are combined with the and operator
+        :param filter:
+        :return: filtered dataframe
+        """
+        cell_idx = np.ones((self.cell_count,), dtype=bool)
+        for key, value in filter.items():
+            if value["variable_type"] == "categorical":
+                key_idx = np.in1d(getattr(self.data.obs, key), value["query"])
+                cell_idx = np.logical_and(cell_idx, key_idx)
+            else:
+                min_ = value["query"]["min"]
+                max_ = value["query"]["max"]
+                if min_ is not None:
+                    key_idx = np.array((getattr(self.data.obs, key) >= min_).data)
+                    cell_idx = np.logical_and(cell_idx, key_idx)
+                if max_ is not None:
+                    key_idx = np.array((getattr(self.data.obs, key) <= max_).data)
+                    cell_idx = np.logical_and(cell_idx, key_idx)
+        return self.data[cell_idx, :]
+
+    def metadata_ranges(self, df=None):
+        metadata_ranges = {}
+        if not df:
+            df = self.data
+        for field in self.schema:
+            if self.schema[field]["variabletype"] == "categorical":
+                group_by = field
+                if group_by == "CellName":
+                    group_by = "cell_name"
+                metadata_ranges[field] = {"options": df.obs.groupby(group_by).size().to_dict()}
+            else:
+                metadata_ranges[field] = {
+                    "range": {
+                        "min": df.obs[field].min(),
+                        "max": df.obs[field].max()
+                    }
+                }
+        return metadata_ranges
+
+    def metadata(self, df, fields=None):
+        """
+         Gets metadata key:value for each cells
+
+        :param df: from filter_cells, dataframe
+        :param fields: list of keys for metadata to return, returns all metadata values if not set.
+        :return: list of metadata values
+        """
+        metadata = df.obs.to_dict(orient="records")
+        for idx in range(len(metadata)):
+            metadata[idx]["CellName"] = metadata[idx].pop("cell_name", None)
+        return metadata
+
+    def create_graph(self, df):
+        """
+        Computes a n-d layout for cells through dimensionality reduction.
+        :param df: from filter_cells, dataframe
+        :return:  [cellid, x, y]
+        """
+        getattr(sc.tl, self.graph_method)(df, random_state=123)
+        graph = df.obsm["X_{graph_method}".format(graph_method=self.graph_method)]
+        normalized_graph = (graph - graph.min()) / (graph.max() - graph.min())
+        return np.hstack((df.obs["cell_name"].values.reshape(len(df.obs.index), 1), normalized_graph)).tolist()
+
+    def diffexp(self, cell_list_1, cell_list_2, pval, num_genes):
+        """
+        Computes the top differentially expressed genes between two clusters
+        :param df1: from filter_cells, dataframe containing first set of cells
+        :param df2: from filter_cells, dataframe containing second set of cells
+        :return: top genes, stats and expression values for top genes
+        """
+        cells_idx_1 = np.in1d(self.data.obs["cell_name"], cell_list_1)
+        cells_idx_2 = np.in1d(self.data.obs["cell_name"], cell_list_2)
+        expression_1 = self.data.X[cells_idx_1, :]
+        expression_2 = self.data.X[cells_idx_2, :]
+        diff_exp = stats.ttest_ind(expression_1, expression_2)
+        # TODO break this up into functions
+        set1 = np.logical_and(diff_exp.pvalue < pval, diff_exp.statistic > 0)
+        set2 = np.logical_and(diff_exp.pvalue < pval, diff_exp.statistic < 0)
+        stat1 = diff_exp.statistic[set1]
+        stat2 = diff_exp.statistic[set2]
+        sort_set1 = np.argsort(stat1)[::-1]
+        sort_set2 = np.argsort(stat2)
+        pval1 = diff_exp.pvalue[set1][sort_set1]
+        pval2 = diff_exp.pvalue[set2][sort_set2]
+        mean_ex1_set1 = np.mean(expression_1[:, set1], axis=0)[sort_set1]
+        mean_ex2_set1 = np.mean(expression_2[:, set1], axis=0)[sort_set1]
+        mean_ex1_set2 = np.mean(expression_1[:, set2], axis=0)[sort_set2]
+        mean_ex2_set2 = np.mean(expression_2[:, set2], axis=0)[sort_set2]
+        mean_diff1 = mean_ex1_set1 - mean_ex2_set1
+        mean_diff2 = mean_ex1_set2 - mean_ex2_set2
+        genes_cellset_1 = self.data.var_names[set1][sort_set1]
+        genes_cellset_2 = self.data.var_names[set2][sort_set2]
+        return {
+            "celllist1": {
+                "topgenes": genes_cellset_1.tolist()[:num_genes],
+                "mean_expression_cellset1": mean_ex1_set1.tolist()[:num_genes],
+                "mean_expression_cellset2": mean_ex2_set1.tolist()[:num_genes],
+                "pval": pval1.tolist()[:num_genes],
+                "ave_diff": mean_diff1.tolist()[:num_genes]
+            },
+            "celllist2": {
+                "topgenes": genes_cellset_2.tolist()[:num_genes],
+                "mean_expression_cellset1": mean_ex1_set2.tolist()[:num_genes],
+                "mean_expression_cellset2": mean_ex2_set2.tolist()[:num_genes],
+                "pval": pval2.tolist()[:num_genes],
+                "ave_diff": mean_diff2.tolist()[:num_genes]
+            },
+        }
+
+    def expression(self, cells=None, genes=None):
+        """
+        Retrieves expression for each gene for cells in data frame
+        :param df:
+        :return: {
+            "genes": list of genes,
+            "cells": list of cells and expression list,
+            "nonzero_gene_count": number of nonzero genes
+        }
+        """
+        if cells:
+            cells_idx = np.in1d(self.data.obs["cell_name"], cells)
+        else:
+            cells_idx = np.ones((self.cell_count,), dtype=bool)
+        if genes:
+            genes_idx = np.in1d(self.data.var_names, genes)
+        else:
+            genes_idx = np.ones((self.gene_count,), dtype=bool)
+        index = np.ix_(cells_idx, genes_idx)
+        expression = self.data.X[index]
+
+        if not genes:
+            genes = self.data.var.index.tolist()
+        if not cells:
+            cells = self.data.obs["cell_name"].tolist()
+
+        cell_data = []
+        for idx, cell in enumerate(cells):
+            cell_data.append({
+                "cellname": cell,
+                "e": list(expression[idx]),
+            })
+
+        return {
+            "genes": genes,
+            "cells": cell_data,
+            "nonzero_gene_count": int(np.sum(expression.any(axis=0)))
+        }