Merge branch 'master' into webgl-pause

.gitignore

@@ -14,3 +14,33 @@ npm-debug.log
 .vscode
 
 data
+
+
+*.idea*
+
+__pycache__
+*.DS_Store*
+
+# Elastic Beanstalk Files
+.elasticbeanstalk/*
+!.elasticbeanstalk/*.cfg.yml
+!.elasticbeanstalk/*.global.yml
+
+GBM
+venv
+extesting
+
+Dockerfile-*
+*-data/*
+data/*
+runServer.py
+templates/favicon.png
+templates/index.html
+templates/service-worker.js
+templates/static/*
+Graph.dot*
+
+server/app/web/static/css/
+server/app/web/static/img/
+server/app/web/static/js/
+server/app/web/templates/index\.html

README.md

@@ -3,7 +3,7 @@
 A React + Redux web application for exploring large scale single cell RNA sequence data.
 
 ##### Quickstart:
-
+* `cd client`
 * `npm install`
 * `npm start`
-* `localhost:3000`
+* `localhost:3000`

client/__tests__/util/bitArray.test.js

@@ -0,0 +1,184 @@
+// jshint esversion: 6
+
+const BitArray = require("../../src/util/typedCrossfilter/bitArray");
+const defaultTestLength = 8;
+
+describe("default select state", () => {
+  test("newly created Bitarray should be deselected", () => {
+    const ba = new BitArray(defaultTestLength);
+    expect(ba).toBeDefined();
+
+    for (let i = 0; i < defaultTestLength; i++) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+
+    const dim = ba.allocDimension();
+    expect(dim).toBeDefined();
+    for (let i = 0; i < defaultTestLength; i++) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+
+    ba.freeDimension(dim);
+    for (let i = 0; i < defaultTestLength; i++) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+  });
+});
+
+describe("select and deselect", () => {
+  test("selectAll and deselectAll", () => {
+    const ba = new BitArray(defaultTestLength);
+    expect(ba).toBeDefined();
+    const dim1 = ba.allocDimension();
+    expect(dim1).toBeDefined();
+    ba.selectAll(dim1);
+
+    for (let i = 0; i < defaultTestLength; i++) {
+      expect(ba.isSelected(i)).toEqual(true);
+    }
+
+    const dim2 = ba.allocDimension();
+    expect(dim2).toBeDefined();
+    for (let i = 0; i < defaultTestLength; i++) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+
+    ba.selectAll(dim2);
+    for (let i = 0; i < defaultTestLength; i++) {
+      expect(ba.isSelected(i)).toEqual(true);
+    }
+
+    ba.deselectAll(dim1);
+    for (let i = 0; i < defaultTestLength; i++) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+
+    ba.deselectAll(dim2);
+    for (let i = 0; i < defaultTestLength; i++) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+
+    ba.selectAll(dim1);
+    ba.selectAll(dim2);
+    for (let i = 0; i < defaultTestLength; i++) {
+      expect(ba.isSelected(i)).toEqual(true);
+    }
+
+    ba.freeDimension(dim1);
+    for (let i = 0; i < defaultTestLength; i++) {
+      expect(ba.isSelected(i)).toEqual(true);
+    }
+
+    ba.freeDimension(dim2);
+    for (let i = 0; i < defaultTestLength; i++) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+  });
+
+  test("selectOne and deselectOne", () => {
+    const ba = new BitArray(defaultTestLength);
+    expect(ba).toBeDefined();
+    const dim = ba.allocDimension();
+    expect(dim).toBeDefined();
+
+    ba.selectOne(dim, 0);
+    expect(ba.isSelected(0)).toEqual(true);
+    for (let i = 1; i < defaultTestLength; i++) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+
+    ba.deselectOne(dim, 0);
+    for (let i = 0; i < defaultTestLength; i++) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+
+    ba.selectOne(dim, 1);
+    expect(ba.isSelected(1)).toEqual(true);
+    expect(ba.isSelected(0)).toEqual(false);
+    for (let i = 2; i < defaultTestLength; i++) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+
+    ba.selectAll(dim);
+    ba.deselectOne(dim, defaultTestLength - 1);
+    expect(ba.isSelected(defaultTestLength - 1)).toEqual(false);
+    for (let i = 0; i < defaultTestLength - 1; i++) {
+      expect(ba.isSelected(i)).toEqual(true);
+    }
+  });
+});
+
+describe("selectionCount", () => {
+  test("simple", () => {
+    const ba = new BitArray(defaultTestLength);
+    expect(ba).toBeDefined();
+    const dim1 = ba.allocDimension();
+    expect(dim1).toBeDefined();
+    const dim2 = ba.allocDimension();
+    expect(dim2).toBeDefined();
+
+    expect(ba.selectionCount).toEqual(0);
+    ba.selectAll(dim1);
+    expect(ba.selectionCount).toEqual(0);
+    ba.selectAll(dim2);
+    expect(ba.selectionCount).toEqual(defaultTestLength);
+
+    for (let i = 0; i < defaultTestLength; i++) {
+      ba.deselectOne(dim1, i);
+      expect(ba.selectionCount).toEqual(defaultTestLength - i - 1);
+      expect(ba.selectionCount).toEqual(ba.countAllOnes());
+    }
+
+    ba.freeDimension(dim1);
+    ba.freeDimension(dim2);
+  });
+});
+
+describe("fillBySelection", () => {
+  test("sets values correctly", () => {
+    const ba = new BitArray(defaultTestLength);
+    expect(ba).toBeDefined();
+    const dim = ba.allocDimension();
+    expect(dim).toBeDefined();
+
+    const arr = new Int32Array(defaultTestLength);
+    arr.fill(0);
+    const truth = new Int32Array(defaultTestLength);
+    truth.fill(0);
+
+    // initial state should be deselected
+    ba.fillBySelection(arr, 1, 0);
+    expect(arr).toEqual(expect.not.arrayContaining([1]));
+
+    // selectAll
+    ba.selectAll(dim);
+    ba.fillBySelection(arr, 1, 0);
+    expect(arr).toEqual(expect.not.arrayContaining([0]));
+
+    // deselectOne
+    ba.deselectOne(dim, 3);
+    ba.fillBySelection(arr, 1, 0);
+    truth.fill(1);
+    truth[3] = 0;
+    expect(arr).toEqual(truth);
+
+    // deselectAll
+    ba.deselectAll(dim);
+    ba.fillBySelection(arr, 1, 0);
+    truth.fill(0);
+    expect(arr).toEqual(truth);
+
+    // selectOne
+    ba.selectOne(dim, 5);
+    ba.fillBySelection(arr, 6, 1);
+    truth.fill(1);
+    truth[5] = 6;
+    expect(arr).toEqual(truth);
+
+    // should be deselected after dimension disposal
+    ba.freeDimension(dim);
+    ba.fillBySelection(arr, 3, 9);
+    truth.fill(9);
+    expect(arr).toEqual(truth);
+  });
+});

client/__tests__/util/positiveInterval.test.js

@@ -0,0 +1,140 @@
+// jshint esversion: 6
+
+const PositiveIntervals = require("../../src/util/typedCrossfilter/positiveIntervals");
+
+describe("canonicalize", () => {
+  test("empty", () => {
+    expect(PositiveIntervals.canonicalize([])).toEqual([]);
+  });
+
+  test("simple, already correct", () => {
+    expect(PositiveIntervals.canonicalize([[0, 1]])).toEqual([[0, 1]]);
+    expect(PositiveIntervals.canonicalize([[0, 1], [2, 3]])).toEqual([
+      [0, 1],
+      [2, 3]
+    ]);
+  });
+
+  test("non-canonical, need to be canonicalized", () => {
+    expect(PositiveIntervals.canonicalize([[0, 1], [1, 2]])).toEqual([[0, 2]]);
+    expect(PositiveIntervals.canonicalize([[1, 2], [2, 3]])).toEqual([[1, 3]]);
+  });
+});
+
+describe("union", () => {
+  test("empty range", () => {
+    expect(PositiveIntervals.union([], [])).toEqual([]);
+    expect(PositiveIntervals.union([], [[1, 2]])).toEqual([[1, 2]]);
+    expect(PositiveIntervals.union([], [[1, 2], [3, 4]])).toEqual([
+      [1, 2],
+      [3, 4]
+    ]);
+    expect(PositiveIntervals.union([[3, 4]], [])).toEqual([[3, 4]]);
+    expect(PositiveIntervals.union([[1, 2], [3, 4]], [])).toEqual([
+      [1, 2],
+      [3, 4]
+    ]);
+    expect(PositiveIntervals.union([[3, 3]], [])).toEqual([[3, 3]]);
+    expect(PositiveIntervals.union([], [[3, 3]])).toEqual([[3, 3]]);
+  });
+
+  test("simple ranges", () => {
+    expect(PositiveIntervals.union([[1, 2]], [[2, 3]])).toEqual([[1, 3]]);
+    expect(PositiveIntervals.union([[2, 3]], [[1, 2]])).toEqual([[1, 3]]);
+    expect(PositiveIntervals.union([[1, 2]], [[3, 4]])).toEqual([
+      [1, 2],
+      [3, 4]
+    ]);
+    expect(
+      PositiveIntervals.union([[1, 2], [3, 4]], [[6, 7], [19, 40]])
+    ).toEqual([[1, 2], [3, 4], [6, 7], [19, 40]]);
+    expect(PositiveIntervals.union([[1, 4]], [[1, 1], [3, 4]])).toEqual([
+      [1, 4]
+    ]);
+    expect(PositiveIntervals.union([[3, 3]], [[4, 4]])).toEqual([
+      [3, 3],
+      [4, 4]
+    ]);
+  });
+});
+
+describe("intersection", () => {
+  test("empty range", () => {
+    expect(PositiveIntervals.intersection([], [])).toEqual([]);
+    expect(PositiveIntervals.intersection([], [[1, 2]])).toEqual([]);
+    expect(PositiveIntervals.intersection([[1, 2]], [])).toEqual([]);
+  });
+
+  test("simple", () => {
+    expect(PositiveIntervals.intersection([[1, 2]], [[2, 3]])).toEqual([]);
+    expect(PositiveIntervals.intersection([[2, 3]], [[1, 2]])).toEqual([]);
+    expect(PositiveIntervals.intersection([[1, 10]], [[1, 10]])).toEqual([
+      [1, 10]
+    ]);
+    expect(PositiveIntervals.intersection([[1, 10]], [[2, 8]])).toEqual([
+      [2, 8]
+    ]);
+    expect(PositiveIntervals.intersection([[2, 8]], [[1, 10]])).toEqual([
+      [2, 8]
+    ]);
+    expect(PositiveIntervals.intersection([[1, 10]], [[2, 12]])).toEqual([
+      [2, 10]
+    ]);
+    expect(PositiveIntervals.intersection([[2, 12]], [[1, 10]])).toEqual([
+      [2, 10]
+    ]);
+    expect(PositiveIntervals.intersection([[1, 10]], [[1, 8]])).toEqual([
+      [1, 8]
+    ]);
+    expect(PositiveIntervals.intersection([[1, 8]], [[1, 10]])).toEqual([
+      [1, 8]
+    ]);
+    expect(PositiveIntervals.intersection([[1, 10]], [[1, 2], [6, 9]])).toEqual(
+      [[1, 2], [6, 9]]
+    );
+    expect(PositiveIntervals.intersection([[0, 2638]], [[1363, 2638]])).toEqual(
+      [[1363, 2638]]
+    );
+    expect(PositiveIntervals.intersection([[1, 2]], [[1, 2]])).toEqual([
+      [1, 2]
+    ]);
+  });
+});
+
+describe("difference", () => {
+  test("empty", () => {
+    expect(PositiveIntervals.difference([], [])).toEqual([]);
+    expect(PositiveIntervals.difference([], [[1, 10]])).toEqual([]);
+    expect(PositiveIntervals.difference([[1, 10]], [])).toEqual([[1, 10]]);
+  });
+
+  test("simple", () => {
+    expect(PositiveIntervals.difference([[1, 2], [3, 4]], [])).toEqual([
+      [1, 2],
+      [3, 4]
+    ]);
+    expect(PositiveIntervals.difference([[1, 2], [3, 10]], [[5, 10]])).toEqual([
+      [1, 2],
+      [3, 5]
+    ]);
+    expect(PositiveIntervals.difference([[1, 2], [3, 10]], [[0, 5]])).toEqual([
+      [5, 10]
+    ]);
+    expect(
+      PositiveIntervals.difference([[0, 2638]], [[0, 1363], [2055, 2638]])
+    ).toEqual([[1363, 2055]]);
+    expect(
+      PositiveIntervals.difference([[0, 1363], [2055, 2638]], [[0, 2638]])
+    ).toEqual([]);
+    expect(PositiveIntervals.difference([[0, 10]], [[0, 1]])).toEqual([
+      [1, 10]
+    ]);
+    expect(PositiveIntervals.difference([[0, 10]], [[1, 2]])).toEqual([
+      [0, 1],
+      [2, 10]
+    ]);
+    expect(PositiveIntervals.difference([[0, 10]], [[9, 10]])).toEqual([
+      [0, 9]
+    ]);
+  });
+});

client/__tests__/util/typedCrossfilter.test.js

@@ -0,0 +1,276 @@
+// jshint esversion: 6
+const _ = require("lodash");
+const crossfilter = require("../../src/util/typedCrossfilter");
+
+const someData = [
+  {
+    date: "2011-11-14T16:17:54Z",
+    quantity: 2,
+    total: 190,
+    tip: 100,
+    type: "tab",
+    productIDs: ["001"]
+  },
+  {
+    date: "2011-11-14T16:20:19Z",
+    quantity: 2,
+    total: 190,
+    tip: 100,
+    type: "tab",
+    productIDs: ["001", "005"]
+  },
+  {
+    date: "2011-11-14T16:28:54Z",
+    quantity: 1,
+    total: 300,
+    tip: 200,
+    type: "visa",
+    productIDs: ["004", "005"]
+  },
+  {
+    date: "2011-11-14T16:30:43Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["001", "002"]
+  },
+  {
+    date: "2011-11-14T16:48:46Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["005"]
+  },
+  {
+    date: "2011-11-14T16:53:41Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["001", "004", "005"]
+  },
+  {
+    date: "2011-11-14T16:54:06Z",
+    quantity: 1,
+    total: 100,
+    tip: 0,
+    type: "cash",
+    productIDs: ["001", "002", "003", "004", "005"]
+  },
+  {
+    date: "2011-11-14T16:58:03Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["001"]
+  },
+  {
+    date: "2011-11-14T17:07:21Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["004", "005"]
+  },
+  {
+    date: "2011-11-14T17:22:59Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["001", "002", "004", "005"]
+  },
+  {
+    date: "2011-11-14T17:25:45Z",
+    quantity: 2,
+    total: 200,
+    tip: 0,
+    type: "cash",
+    productIDs: ["002"]
+  },
+  {
+    date: "2011-11-14T17:29:52Z",
+    quantity: 1,
+    total: 200,
+    tip: 100,
+    type: "visa",
+    productIDs: ["004"]
+  }
+];
+
+var payments = null;
+beforeEach(() => {
+  payments = crossfilter(someData);
+});
+
+describe("typedCrossfilter", () => {
+  test("alloc and free", () => {
+    expect(payments).toBeDefined();
+    expect(payments.size()).toEqual(someData.length);
+    expect(payments.all()).toEqual(someData);
+
+    const quantity = payments.dimension(r => r.quantity, Int32Array);
+    expect(quantity).toBeDefined();
+    expect(quantity.id()).toBeDefined();
+
+    quantity.dispose();
+    expect(payments.size()).toEqual(someData.length);
+    expect(payments.all()).toEqual(someData);
+  });
+
+  test("filterAll and filterNone", () => {
+    expect(payments).toBeDefined();
+    const quantity = payments.dimension(r => r.quantity, Int32Array);
+    const tip = payments.dimension(r => r.tip, Float32Array);
+    const total = payments.dimension(r => r.total, Float32Array);
+    const type = payments.dimension(r => r.type, "enum");
+
+    expect(quantity).toBeDefined();
+    expect(tip).toBeDefined();
+    expect(total).toBeDefined();
+    expect(type).toBeDefined();
+
+    // initially, all should be filtered
+    expect(payments.allFiltered().length).toEqual(payments.size());
+    expect(payments.allFiltered()).toEqual(payments.all());
+    expect(payments.countFiltered()).toEqual(someData.length);
+
+    // filterAll
+    tip.filterAll(); // should change nothing
+    expect(payments.allFiltered()).toEqual(payments.all());
+    expect(payments.countFiltered()).toEqual(someData.length);
+
+    // ditto
+    total.filterAll();
+    expect(payments.allFiltered()).toEqual(payments.all());
+    expect(payments.countFiltered()).toEqual(someData.length);
+
+    // filterNone
+    type.filterNone();
+    expect(payments.allFiltered()).toEqual([]);
+    expect(payments.countFiltered()).toEqual(0);
+
+    quantity.filterNone();
+    expect(payments.allFiltered()).toEqual([]);
+    expect(payments.countFiltered()).toEqual(0);
+
+    // invert the first none; should have no effect because type is
+    // still not filtered
+    quantity.filterAll();
+    expect(payments.allFiltered()).toEqual([]);
+    expect(payments.countFiltered()).toEqual(0);
+
+    // filter all of type; should select all
+    type.filterAll();
+    expect(payments.allFiltered()).toEqual(payments.all());
+    expect(payments.countFiltered()).toEqual(payments.size());
+  });
+
+  test("filterExact", () => {
+    expect(payments).toBeDefined();
+    const quantity = payments.dimension(r => r.quantity, Int32Array);
+    const tip = payments.dimension(r => r.tip, Float32Array);
+    const total = payments.dimension(r => r.total, Float32Array);
+    const type = payments.dimension(r => r.type, "enum");
+
+    quantity.filterExact(1);
+    expect(payments.countFiltered()).toEqual(
+      _.countBy(someData, "quantity")[1]
+    );
+    expect(payments.allFiltered()).toEqual(_.filter(someData, { quantity: 1 }));
+
+    tip.filterExact(0);
+    expect(payments.allFiltered()).toEqual(
+      _.filter(someData, { tip: 0, quantity: 1 })
+    );
+
+    type.filterExact("cash");
+    expect(payments.allFiltered()).toEqual(
+      _.filter(someData, { tip: 0, quantity: 1, type: "cash" })
+    );
+  });
+
+  test("filterRange", () => {
+    expect(payments).toBeDefined();
+    const quantity = payments.dimension(r => r.quantity, Int32Array);
+    const tip = payments.dimension(r => r.tip, Float32Array);
+    const total = payments.dimension(r => r.total, Float32Array);
+    const type = payments.dimension(r => r.type, "enum");
+
+    tip.filterRange([0, 91]);
+    expect(payments.allFiltered()).toEqual(
+      _(someData)
+        .filter(r => r.tip >= 0 && r.tip < 91)
+        .value()
+    );
+
+    tip.filterRange([0, 90]);
+    expect(payments.allFiltered()).toEqual(
+      _(someData)
+        .filter(r => r.tip >= 0 && r.tip < 90)
+        .value()
+    );
+
+    tip.filterRange([1, 90]);
+    expect(payments.allFiltered()).toEqual(
+      _(someData)
+        .filter(r => r.tip >= 1 && r.tip < 91)
+        .value()
+    );
+  });
+
+  test("filterEnum", () => {
+    expect(payments).toBeDefined();
+    const quantity = payments.dimension(r => r.quantity, Int32Array);
+    const tip = payments.dimension(r => r.tip, Float32Array);
+    const total = payments.dimension(r => r.total, Float32Array);
+    const type = payments.dimension(r => r.type, "enum");
+
+    type.filterEnum(["tab", "cash"]);
+    expect(payments.allFiltered()).toEqual(
+      _(someData)
+        .filter(r => r.type === "cash" || r.type === "tab")
+        .value()
+    );
+
+    tip.filterEnum([0, 100]);
+    expect(payments.allFiltered()).toEqual(
+      _(someData)
+        .filter(r => r.type === "cash" || r.type === "tab")
+        .filter(r => r.tip === 0 || r.tip === 100)
+        .value()
+    );
+  });
+
+  test("more than 32 dimensions", () => {
+    expect(payments).toBeDefined();
+    const quantity = payments.dimension(r => r.quantity, Int32Array);
+    const tip = payments.dimension(r => r.tip, Float32Array);
+    const total = payments.dimension(r => r.total, Float32Array);
+    const type = payments.dimension(r => r.type, "enum");
+
+    // Create a bunch of fake dimensions to ensure we can handle > 32
+    let dimMap = {};
+    for (let i = 0; i < 65; i++) {
+      dimMap[i] = payments.dimension(r => Math.random(), Float32Array);
+      expect(dimMap[i]).toBeDefined();
+      expect(dimMap[i].id()).toBeDefined();
+    }
+
+    // everything should start as selected/filtered
+    expect(payments.countFiltered()).toEqual(someData.length);
+
+    dimMap[0].filterAll();
+    dimMap[64].filterAll();
+    expect(payments.countFiltered()).toEqual(someData.length);
+
+    dimMap[33].filterNone();
+    expect(payments.allFiltered()).toEqual([]);
+
+    dimMap[33].filterAll();
+    expect(payments.allFiltered()).toEqual(someData);
+  });
+});

client/package.json

@@ -25,6 +25,9 @@
     "sourceMap": false,
     "instrument": false
   },
+  "resolutions": {
+    "eslint-scope": "3.7.1"
+  },
   "dependencies": {
     "canvas-fit": "^1.5.0",
     "d3": "^4.10.0",
@@ -112,6 +115,8 @@
     "whatwg-fetch": "^2.0.1"
   },
   "jest": {
-    "testMatch": ["**/__tests__/**/?(*.)(spec|test).js?(x)"]
+    "testMatch": [
+      "**/__tests__/**/?(*.)(spec|test).js?(x)"
+    ]
   }
 }

client/src/reducers/controls.js

@@ -2,7 +2,7 @@
 import _ from "lodash";
 import { parseRGB } from "../util/parseRGB";
 import { createSchemaByDataSniffing } from "../util/schema";
-var crossfilter = require("../util/typedCrossfilter");
+import crossfilter from "../util/typedCrossfilter";
 
 // Deduce the correct crossfilter dimension type from a metadata
 // schema description.

client/src/util/typedCrossfilter/bitArray.js

@@ -226,4 +226,4 @@ class BitArray {
   }
 }
 
-module.exports = BitArray;
+export default BitArray;

client/src/util/typedCrossfilter/index.js

@@ -29,9 +29,9 @@ more complex API.  In a few cases, elements of that API were incorporated.
 
 */
 
-var PositiveIntervals = require("./positiveIntervals");
+import PositiveIntervals from "./positiveIntervals";
-var BitArray = require("./bitArray");
+import BitArray from "./bitArray";
-var Util = require("./util");
+import {fillRange, lowerBound, lowerBoundIndirect, upperBound, upperBoundIndirect} from "./util";
 
 class TypedCrossfilter {
   constructor(data) {
@@ -118,7 +118,7 @@ class ScalarDimension {
     this.value = array;
 
     // create sort index
-    this.index = Util.fillRange(new Uint32Array(this.crossfilter.data.length));
+    this.index = fillRange(new Uint32Array(this.crossfilter.data.length));
     this.index.sort((a, b) => array[a] - array[b]);
   }
 
@@ -179,14 +179,14 @@ class ScalarDimension {
   // filter by value - exact match
   filterExact(value) {
     const newFilter = [
-      Util.lowerBoundIndirect(
+      lowerBoundIndirect(
         this.value,
         this.index,
         value,
         0,
         this.value.length
       ),
-      Util.upperBoundIndirect(
+      upperBoundIndirect(
         this.value,
         this.index,
         value,
@@ -207,14 +207,14 @@ class ScalarDimension {
     const newFilter = [];
     for (let v = 0, len = values.length; v < len; v++) {
       const intv = [
-        Util.lowerBoundIndirect(
+        lowerBoundIndirect(
           this.value,
           this.index,
           values[v],
           0,
           this.value.length
         ),
-        Util.upperBoundIndirect(
+        upperBoundIndirect(
           this.value,
           this.index,
           values[v],
@@ -233,14 +233,14 @@ class ScalarDimension {
   filterRange(range) {
     const newFilter = [];
     const intv = [
-      Util.lowerBoundIndirect(
+      lowerBoundIndirect(
         this.value,
         this.index,
         range[0],
         0,
         this.value.length
       ),
-      Util.upperBoundIndirect(
+      upperBoundIndirect(
         this.value,
         this.index,
         range[1],
@@ -350,7 +350,7 @@ class EnumDimension extends ScalarDimension {
     const enumLen = this.enumIndex.length;
     for (let i = 0; i < len; i++) {
       const v = value(data[i]);
-      const e = Util.lowerBound(this.enumIndex, v, 0, enumLen);
+      const e = lowerBound(this.enumIndex, v, 0, enumLen);
       array[i] = e;
     }
     return array;
@@ -358,14 +358,14 @@ class EnumDimension extends ScalarDimension {
 
   filterExact(value) {
     return super.filterExact(
-      Util.lowerBound(this.enumIndex, value, 0, this.enumIndex.length)
+      lowerBound(this.enumIndex, value, 0, this.enumIndex.length)
     );
   }
 
   filterEnum(values) {
     return super.filterEnum(
       values.map(v =>
-        Util.lowerBound(this.enumIndex, v, 0, this.enumIndex.length)
+        lowerBound(this.enumIndex, v, 0, this.enumIndex.length)
       )
     );
   }
@@ -373,7 +373,7 @@ class EnumDimension extends ScalarDimension {
   filterRange(range) {
     return super.filterEnum(
       range.map(v =>
-        Util.lowerBound(this.enumIndex, v, 0, this.enumIndex.length)
+        lowerBound(this.enumIndex, v, 0, this.enumIndex.length)
       )
     );
   }
@@ -391,4 +391,4 @@ crossfilter.TypedCrossfilter = TypedCrossfilter;
 crossfilter.ScalarDimension = ScalarDimension;
 crossfilter.EnumDimension = EnumDimension;
 
-module.exports = crossfilter;
+export default crossfilter;

client/src/util/typedCrossfilter/positiveIntervals.js

@@ -129,4 +129,4 @@ class PositiveIntervals {
   }
 }
 
-module.exports = PositiveIntervals;
+export default PositiveIntervals;

client/src/util/typedCrossfilter/util.js

@@ -8,7 +8,7 @@
 // fill an array or typedarray with a sequential range of numbers,
 // starting with `start`
 //
-function fillRange(arr, start = 0) {
+export function fillRange(arr, start = 0) {
   for (let i = 0, len = arr.length; i < len; i++) {
     arr[i] = i + start;
   }
@@ -30,7 +30,7 @@ function fillRange(arr, start = 0) {
 // a factory version of lowerBound that takes an accessor (rather than having
 // a special-cased version for lining the indirection).
 //
-function lowerBound(valueArray, value, first, last) {
+export function lowerBound(valueArray, value, first, last) {
   // this is just a binary search
   while (first < last) {
     const middle = (first + last) >>> 1;
@@ -45,7 +45,7 @@ function lowerBound(valueArray, value, first, last) {
 
 // Inlined performance optimization - used to indirect through a sort map.
 //
-function lowerBoundIndirect(valueArray, indexArray, value, first, last) {
+export function lowerBoundIndirect(valueArray, indexArray, value, first, last) {
   // this is just a binary search
   while (first < last) {
     const middle = (first + last) >>> 1;
@@ -69,7 +69,7 @@ function lowerBoundIndirect(valueArray, indexArray, value, first, last) {
 //    C++: upper_bound()
 //    Python: bisect.bisect_right()
 //
-function upperBound(valueArray, value, first, last) {
+export function upperBound(valueArray, value, first, last) {
   // this is just a binary search
   while (first < last) {
     const middle = (first + last) >>> 1;
@@ -84,7 +84,7 @@ function upperBound(valueArray, value, first, last) {
 
 // Inline performance optimization
 //
-function upperBoundIndirect(valueArray, indexArray, value, first, last) {
+export function upperBoundIndirect(valueArray, indexArray, value, first, last) {
   // this is just a binary search
   while (first < last) {
     const middle = (first + last) >>> 1;
@@ -96,11 +96,3 @@ function upperBoundIndirect(valueArray, indexArray, value, first, last) {
   }
   return first;
 }
-
-module.exports = {
-  fillRange,
-  lowerBound,
-  lowerBoundIndirect,
-  upperBound,
-  upperBoundIndirect
-};

server/app/__init__.py

@@ -0,0 +1,55 @@
+import os
+
+from flask import Flask
+from flask_compress import Compress
+from flask_cors import CORS
+from flask_restful_swagger_2 import get_swagger_blueprint
+
+from .web import webapp
+from .rest_api.rest import get_api_resources
+
+app = Flask(__name__)
+Compress(app)
+CORS(app)
+
+# Config
+CXG_DIR = os.environ.get("CXG_DIRECTORY", default="/Users/charlotteweaver/Documents/Git/cxg-v2/data/")
+SECRET_KEY = os.environ.get("CXG_SECRET_KEY", default="SparkleAndShine")
+ENGINE = os.environ.get("CXG_ENGINE", default="scanpy")
+TITLE = os.environ.get("DATASET_TITLE", default="PBMC 3K")
+# TODO remove the 2 when this is prod
+CXG_API_BASE = os.environ.get("CXG_API_BASE2", default="http://0.0.0.0:5005/api/")
+
+app.config.update(
+    SECRET_KEY=SECRET_KEY,
+    CXG_API_BASE=CXG_API_BASE,
+    ENGINE=ENGINE,
+    DATA=CXG_DIR,
+    DATASET_TITLE=TITLE
+)
+
+app.config["PROFILE"] = True
+# app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[15])
+
+# Application Data
+data = None
+if app.config["ENGINE"] == "scanpy":
+    from .scanpy_engine.scanpy_engine import ScanpyEngine
+    data = ScanpyEngine(app.config["DATA"], schema="data_schema.json")
+
+REACTIVE_LIMIT = 1_000_000
+
+# A list of swagger document objects
+docs = []
+resources = get_api_resources()
+docs.append(resources.get_swagger_doc())
+
+
+app.register_blueprint(webapp.bp)
+app.register_blueprint(resources.blueprint)
+app.register_blueprint(
+    get_swagger_blueprint(docs, "/api/swagger", produces=["application/json"], title="cellxgene rest api",
+                          description="An API connecting ExpressionMatrix2 clustering algorithm to cellxgene"))
+
+
+app.add_url_rule("/", endpoint="index")

server/app/driver/driver.py

@@ -0,0 +1,81 @@
+from abc import ABCMeta, abstractmethod
+
+
+class CXGDriver(metaclass=ABCMeta):
+    def __init__(self, data, schema=None, graph_method=None, diffexp_method=None):
+        self.data = self._load_data(data)
+
+    @staticmethod
+    @abstractmethod
+    def _load_data(data):
+        pass
+
+    @abstractmethod
+    def _load_or_infer_schema(data):
+        pass
+
+    @abstractmethod
+    def cells(self):
+        pass
+
+    @abstractmethod
+    def genes(self):
+        pass
+
+    @abstractmethod
+    def filter_cells(self, filter):
+        """
+        Filter cells from data and return a subset of the data
+        A filter is a dictionary where the key is a metadatata category
+        Value is dictionary
+            value_type: int, float, string
+            variable_type: continuous, categorical
+            query: filter value, for categorical [val1, val2], for continuous {min: x, max:y}
+        Filters are combined with the and operator
+        :param filter:
+        :return: filtered dataframe
+        """
+        pass
+
+    @abstractmethod
+    def metadata(self, df, fields=None):
+        """
+         Gets metadata key:value for each cells
+
+        :param df: from filter_cells, dataframe
+        :param fields: list of keys for metadata to return, returns all metadata values if not set.
+        :return: list of metadata values
+        """
+        pass
+
+    @abstractmethod
+    def create_graph(self, df):
+        """
+        Computes a n-d layout for cells through dimensionality reduction.
+        :param df: from filter_cells, dataframe
+        :return:  [cellid, x, y]
+        """
+        pass
+
+    @abstractmethod
+    def diffexp(self, df1, df2):
+        """
+        Computes the top differentially expressed genes between two clusters
+        :param df1: from filter_cells, dataframe containing first set of cells
+        :param df2: from filter_cells, dataframe containing second set of cells
+        :return: top genes, stats and expression values for top genes
+        """
+        pass
+
+    @abstractmethod
+    def expression(self, df):
+        """
+        Retrieves expression for each gene for cells in data frame
+        :param df:
+        :return: {
+            "genes": list of genes,
+            "cells": list of cells and expression list,
+            "nonzero_gene_count": number of nonzero genes
+        }
+        """
+        pass

server/app/rest_api/rest.py

@@ -0,0 +1,439 @@
+from flask import (
+    Blueprint, request
+)
+from flask_restful_swagger_2 import Api, swagger, Resource
+
+from ..util.utils import make_payload
+from ..util.filter import parse_filter
+
+
+class InitializeAPI(Resource):
+    @swagger.doc({
+        "summary": "get metadata schema, ranges for values, and cell count to initialize cellxgene app",
+        "tags": ["initialize"],
+        "parameters": [],
+        "responses": {
+            "200": {
+                "description": "initialization data for UI",
+                "examples": {
+                    "application/json": {
+                        "data": {
+                            "cellcount": 3589,
+                            "options": {
+                                "Sample.type": {
+                                    "options": {
+                                        "Glioblastoma": 3589
+                                    }
+                                },
+                                "Selection": {
+                                    "options": {
+                                        "Astrocytes(HEPACAM)": 714,
+                                        "Endothelial(BSC)": 123,
+                                        "Microglia(CD45)": 1108,
+                                        "Neurons(Thy1)": 685,
+                                        "Oligodendrocytes(GC)": 294,
+                                        "Unpanned": 665
+                                    }
+                                },
+                                "Splice_sites_AT.AC": {
+                                    "range": {
+                                        "max": 1025,
+                                        "min": 152
+                                    }
+                                },
+                                "Splice_sites_Annotated": {
+                                    "range": {
+                                        "max": 1075869,
+                                        "min": 26
+                                    }
+                                }
+                            },
+                            "schema": {
+                                "CellName": {
+                                    "displayname": "Name",
+                                    "type": "string",
+                                    "variabletype": "categorical"
+                                },
+                                "Class": {
+                                    "displayname": "Class",
+                                    "type": "string",
+                                    "variabletype": "categorical"
+                                },
+                                "ERCC_reads": {
+                                    "displayname": "ERCC Reads",
+                                    "type": "int",
+                                    "variabletype": "continuous"
+                                },
+                                "ERCC_to_non_ERCC": {
+                                    "displayname": "ERCC:Non-ERCC",
+                                    "type": "float",
+                                    "variabletype": "continuous"
+                                },
+                                "Genes_detected": {
+                                    "displayname": "Genes Detected",
+                                    "type": "int",
+                                    "variabletype": "continuous"
+                                }
+                            },
+                            "genes": ["1/2-SBSRNA4", "A1BG", "A1BG-AS1"]
+
+                        },
+                        "status": {
+                            "error": False,
+                            "errormessage": ""
+                        }
+                    }
+                }
+            }
+        }
+    })
+    def get(self):
+        from app import data, REACTIVE_LIMIT
+        return make_payload({
+            "schema": data.schema,
+            "cellcount": data.cell_count,
+            "reactivelimit": REACTIVE_LIMIT,
+            "genes": data.genes(),
+            "ranges": data.metadata_ranges(),
+
+        })
+
+
+class CellsAPI(Resource):
+    @swagger.doc({
+        "summary": "filter based on metadata fields to get a subset cells, expression data, and metadata",
+        "tags": ["cells"],
+        "description": "Cells takes query parameters defined in the schema retrieved from the /initialize enpoint. "
+                       "<br>For categorical metadata keys filter based on `key=value` <br>"
+                       " For continuous metadata keys filter by `key=min,max`<br> Either value "
+                       "can be replaced by a \*. To have only a minimum value `key=min,\*`  To have only a maximum "
+                       "value `key=\*,max` <br>Graph data (if retrieved) is normalized"
+                       " To only retrieve cells that don't have a value for the key filter by `key`",
+        "parameters": [],
+
+        "responses": {
+            "200": {
+                "description": "initialization data for UI",
+                "examples": {
+                    "application/json": {
+                        "data": {
+                            "badmetadatacount": 0,
+                            "cellcount": 0,
+                            "cellids": ["..."],
+                            "metadata": [
+                                {
+                                    "CellName": "1001000173.G8",
+                                    "Class": "Neoplastic",
+                                    "Cluster_2d": "11",
+                                    "Cluster_2d_color": "#8C564B",
+                                    "Cluster_CNV": "1",
+                                    "Cluster_CNV_color": "#1F77B4",
+                                    "ERCC_reads": "152104",
+                                    "ERCC_to_non_ERCC": "0.562454470489481",
+                                    "Genes_detected": "1962",
+                                    "Location": "Tumor",
+                                    "Location.color": "#FF7F0E",
+                                    "Multimapping_reads_percent": "2.67",
+                                    "Neoplastic": "Neoplastic",
+                                    "Non_ERCC_reads": "270429",
+                                    "Sample.name": "BT_S2",
+                                    "Sample.name.color": "#AEC7E8",
+                                    "Sample.type": "Glioblastoma",
+                                    "Sample.type.color": "#1F77B4",
+                                    "Selection": "Unpanned",
+                                    "Selection.color": "#98DF8A",
+                                    "Splice_sites_AT.AC": "102",
+                                    "Splice_sites_Annotated": "122397",
+                                    "Splice_sites_GC.AG": "761",
+                                    "Splice_sites_GT.AG": "125741",
+                                    "Splice_sites_non_canonical": "56",
+                                    "Splice_sites_total": "126660",
+                                    "Total_reads": "1741039",
+                                    "Unique_reads": "1400382",
+                                    "Unique_reads_percent": "80.43",
+                                    "Unmapped_mismatch": "2.15",
+                                    "Unmapped_other": "0.18",
+                                    "Unmapped_short": "14.56",
+                                    "housekeeping_cluster": "2",
+                                    "housekeeping_cluster_color": "#AEC7E8",
+                                    "recluster_myeloid": "NA",
+                                    "recluster_myeloid_color": "NA"
+                                },
+                            ],
+                            "reactive": True,
+                            "graph": [
+                                [
+                                    "1001000173.G8",
+                                    0.93836,
+                                    0.28623
+                                ],
+
+                                [
+                                    "1001000173.D4",
+                                    0.1662,
+                                    0.79438
+                                ]
+
+                            ],
+                            "status": {
+                                "error": False,
+                                "errormessage": ""
+                            }
+
+                        },
+                    }
+                },
+            },
+
+            "400": {
+                "description": "bad query params",
+            }
+        }
+    })
+    def get(self):
+        from app import data
+        payload = {
+            "metadata": [],
+            "cellcount": 0,
+            "graph": [],
+            "ranges": {},
+        }
+        # get query params
+        filter = parse_filter(request.args, data.schema)
+        filtered_data = data.filter_cells(filter)
+        payload["metadata"] = data.metadata(filtered_data)
+        payload["ranges"] = data.metadata_ranges(filtered_data)
+        payload["graph"] = data.create_graph(filtered_data)
+        payload["cellcount"] = data.cell_count
+        return make_payload(payload)
+
+
+class ExpressionAPI(Resource):
+    @swagger.doc({
+        "summary": "Json with gene list and expression data by cell, limited to first 40 cells",
+        "tags": ["expression"],
+        "parameters": [
+            {
+                "name": "include_unexpressed_genes",
+                "description": "Include genes that have 0 expression across all cells in set",
+                "in": "path",
+                "type": "bool",
+            }
+        ],
+        "responses": {
+            "200": {
+                "description": "Json for heatmap",
+                "examples": {
+                    "application/json": {
+                        "data": {
+                            "cells": [
+                                {
+                                    "cellname": "1/2-SBSRNA4",
+                                    "e": [0, 0, 214, 0, 0]
+                                },
+                            ],
+                            "genes": [
+                                "1001000173.G8",
+                                "1001000173.D4",
+                                "1001000173.B4",
+                                "1001000173.A2",
+                                "1001000173.E2"
+                            ],
+                            "nonzero_gene_count": 2857
+                        },
+                        "status": {
+                            "error": False,
+                            "errormessage": ""
+                        }
+                    }
+                }
+            }
+        }
+    })
+    def get(self):
+        from app import data
+        expression_data = data.expression()
+        return make_payload(expression_data)
+
+    @swagger.doc({
+        "summary": "Json with gene list and expression data by cell",
+        "tags": ["expression"],
+        "parameters": [
+            {
+                "name": "body",
+                "in": "body",
+                "schema": {
+                    "example": {
+                        "celllist": ["1001000173.G8", "1001000173.D4"],
+                        "genelist": ["1/2-SBSRNA4", "A1BG", "A1BG-AS1", "A1CF", "A2LD1", "A2M", "A2ML1", "A2MP1",
+                                     "A4GALT"],
+                        "include_unexpressed_genes": True,
+                    }
+
+                }
+            },
+        ],
+        "responses": {
+            "200": {
+                "description": "Json for expressiondata",
+                "examples": {
+                    "application/json": {
+                        "data": {
+                            "cells": [
+                                {
+                                    "cellname": "1001000173.D4",
+                                    "e": [0, 0]
+                                },
+                                {
+                                    "cellname": "1001000173.G8",
+                                    "e": [0, 0]
+                                }
+                            ],
+                            "genes": [
+                                "ABCD4",
+                                "ZWINT"
+                            ],
+                            "nonzero_gene_count": 2857
+                        },
+                        "status": {
+                            "error": False,
+                            "errormessage": ""
+                        }
+
+                    }
+                }
+            },
+            "400": {
+                "description": "Required parameter missing/incorrect",
+            }
+        }
+    })
+    def post(self):
+        from app import data
+        args = request.get_json()
+        cell_list = args.get("celllist", [])
+        gene_list = args.get("genelist", [])
+        if not cell_list and not gene_list:
+            return make_payload([], "must include celllist and/or genelist parameter", 400)
+
+        expression_data = data.expression(cell_list, gene_list)
+
+        if cell_list and len(expression_data["cells"]) < len(cell_list):
+            return make_payload([], "Some cell ids not available", 400)
+        if gene_list and len(expression_data["genes"]) < len(gene_list):
+            return make_payload([], "Some genes not available", 400)
+
+        return make_payload(expression_data)
+
+
+class DifferentialExpressionAPI(Resource):
+    @swagger.doc({
+        "summary": "Get the top expressed genes for two cell sets. Calculated using t-test",
+        "tags": ["expression"],
+        "parameters": [
+            {
+                "name": "body",
+                "in": "body",
+                "schema": {
+                    "example": {
+                        "celllist1": ["1001000176.C12", "1001000176.C7", "1001000177.F11"],
+                        "celllist2": ["1001000012.D2", "1001000017.F10", "1001000033.C3", "1001000229.D4"],
+                        "num_genes": 5,
+                        "pval": 0.000001,
+                    },
+                }
+            }
+        ],
+        "responses": {
+            "200": {
+                "description": "top expressed genes for cellset1, cellset2",
+                "examples": {
+                    "application/json": {
+                        "data": {
+                            "celllist1": {
+                                "ave_diff": [
+                                    432.0132935431362,
+                                    12470.5623982637,
+                                    957.0246880086814
+                                ],
+                                "mean_expression_cellset1": [
+                                    438.6185567010309,
+                                    13315.536082474227,
+                                    1076.5773195876288
+                                ],
+                                "mean_expression_cellset2": [
+                                    6.605263157894737,
+                                    844.9736842105264,
+                                    119.55263157894737
+                                ],
+                                "pval": [
+                                    3.8906598089944563e-35,
+                                    1.9086226376018916e-25,
+                                    7.847480544069826e-21
+                                ],
+                                "topgenes": [
+                                    "TMSB10",
+                                    "FTL",
+                                    "TMSB4X"
+                                ]
+                            },
+                            "celllist2": {
+                                "ave_diff": [
+                                    -6860.599158979924,
+                                    -519.1314432989691,
+                                    -10278.328269126423
+                                ],
+                                "mean_expression_cellset1": [
+                                    2.8350515463917527,
+                                    0.6185567010309279,
+                                    23.09278350515464
+                                ],
+                                "mean_expression_cellset2": [
+                                    6863.434210526316,
+                                    519.75,
+                                    10301.421052631578
+                                ],
+                                "pval": [
+                                    4.662891833748732e-44,
+                                    3.6278087029927103e-37,
+                                    8.396825170618402e-35
+                                ],
+                                "topgenes": [
+                                    "SPARCL1",
+                                    "C1orf61",
+                                    "CLU"
+                                ]
+                            }
+                        },
+                        "status": {
+                            "error": False,
+                            "errormessage": ""
+                        }
+                    }
+                }
+            }
+        }
+    })
+    def post(self):
+        from app import data
+        args = request.get_json()
+        cell_list_1 = args.get("celllist1", [])
+        cell_list_2 = args.get("celllist2", [])
+        num_genes = args.get("num_genes", 7)
+        pval = args.get("pval", 0.5)
+        if not (cell_list_1 and cell_list_2):
+            return make_payload([],
+                                "must include celllist1 and celllist2 parameters",
+                                400)
+        data = data.diffexp(cell_list_1, cell_list_2, pval, num_genes)
+        return make_payload(data)
+
+
+def get_api_resources():
+    bp = Blueprint("api", __name__, url_prefix="/api/v0.1")
+    api = Api(bp, add_api_spec_resource=False)
+    api.add_resource(InitializeAPI, "/initialize")
+    api.add_resource(CellsAPI, "/cells")
+    api.add_resource(ExpressionAPI, "/expression")
+    api.add_resource(DifferentialExpressionAPI, "/diffexpression")
+    return api

server/app/scanpy_engine/scanpy_engine.py

@@ -0,0 +1,197 @@
+import os
+
+import numpy as np
+import scanpy.api as sc
+from scipy import stats
+
+from ..util.schema_parse import parse_schema
+from ..driver.driver import CXGDriver
+
+
+class ScanpyEngine(CXGDriver):
+
+    def __init__(self, data, schema=None, graph_method="umap", diffexp_method="ttest"):
+        self.data = self._load_data(data)
+        self.schema = self._load_or_infer_schema(data, schema)
+        self._set_cell_names()
+        self.cell_count = self.data.shape[0]
+        self.gene_count = self.data.shape[1]
+        self.graph_method = graph_method
+        self.diffexp_method = diffexp_method
+
+    def _set_cell_names(self):
+        self.data.obs["cell_name"] = list(self.data.obs.index)
+
+    @staticmethod
+    def _load_data(data):
+        return sc.read(os.path.join(data, "data.h5ad"))
+
+    @staticmethod
+    def _load_or_infer_schema(data, schema):
+        data_schema = None
+        if not schema:
+            pass
+        else:
+            data_schema = parse_schema(os.path.join(data, schema))
+        return data_schema
+
+    def cells(self):
+        return list(self.data.obs.index)
+
+    def genes(self):
+        return self.data.var.index.tolist()
+
+    def filter_cells(self, filter):
+        """
+        Filter cells from data and return a subset of the data
+        A filter is a dictionary where the key is a metadatata category
+        Value is dictionary
+            value_type: int, float, string
+            variable_type: continuous, categorical
+            query: filter value, for categorical [val1, val2], for continuous {min: x, max:y}
+        Filters are combined with the and operator
+        :param filter:
+        :return: filtered dataframe
+        """
+        cell_idx = np.ones((self.cell_count,), dtype=bool)
+        for key, value in filter.items():
+            if value["variable_type"] == "categorical":
+                key_idx = np.in1d(getattr(self.data.obs, key), value["query"])
+                cell_idx = np.logical_and(cell_idx, key_idx)
+            else:
+                min_ = value["query"]["min"]
+                max_ = value["query"]["max"]
+                if min_:
+                    key_idx = np.array((getattr(self.data.obs, key) >= min_).data)
+                    cell_idx = np.logical_and(cell_idx, key_idx)
+                if max_:
+                    key_idx = np.array((getattr(self.data.obs, key) <= min_).data)
+                    cell_idx = np.logical_and(cell_idx, key_idx)
+        return self.data[cell_idx, :]
+
+    def metadata_ranges(self, df=None):
+        metadata_ranges = {}
+        if not df:
+            df = self.data
+        for field in self.schema:
+            if self.schema[field]["variabletype"] == "categorical":
+                group_by = field
+                if group_by == "CellName":
+                    group_by = "cell_name"
+                metadata_ranges[field] = {"options": df.obs.groupby(group_by).size().to_dict()}
+            else:
+                metadata_ranges[field] = {
+                    "range": {
+                        "min": df.obs[field].min(),
+                        "max": df.obs[field].max()
+                    }
+                }
+        return metadata_ranges
+
+    def metadata(self, df, fields=None):
+        """
+         Gets metadata key:value for each cells
+
+        :param df: from filter_cells, dataframe
+        :param fields: list of keys for metadata to return, returns all metadata values if not set.
+        :return: list of metadata values
+        """
+        metadata = df.obs.to_dict(orient="records")
+        for idx in range(len(metadata)):
+            metadata[idx]["CellName"] = metadata[idx].pop("cell_name", None)
+        return metadata
+
+    def create_graph(self, df):
+        """
+        Computes a n-d layout for cells through dimensionality reduction.
+        :param df: from filter_cells, dataframe
+        :return:  [cellid, x, y]
+        """
+        getattr(sc.tl, self.graph_method)(df)
+        graph = df.obsm["X_{graph_method}".format(graph_method=self.graph_method)]
+        normalized_graph = (graph - graph.min()) / (graph.max() - graph.min())
+        return np.hstack((df.obs["cell_name"].values.reshape(len(df.obs.index), 1), normalized_graph)).tolist()
+
+    def diffexp(self, cell_list_1, cell_list_2, pval, num_genes):
+        """
+        Computes the top differentially expressed genes between two clusters
+        :param df1: from filter_cells, dataframe containing first set of cells
+        :param df2: from filter_cells, dataframe containing second set of cells
+        :return: top genes, stats and expression values for top genes
+        """
+        cells_idx_1 = np.in1d(self.data.obs["cell_name"], cell_list_1)
+        cells_idx_2 = np.in1d(self.data.obs["cell_name"], cell_list_2)
+        expression_1 = self.data.X[cells_idx_1, :]
+        expression_2 = self.data.X[cells_idx_2, :]
+        diff_exp = stats.ttest_ind(expression_1, expression_2)
+        set1 = np.logical_and(diff_exp.pvalue < pval, diff_exp.statistic > 0)
+        set2 = np.logical_and(diff_exp.pvalue < pval, diff_exp.statistic < 0)
+        stat1 = diff_exp.statistic[set1]
+        stat2 = diff_exp.statistic[set2]
+        sort_set1 = np.argsort(stat1)[::-1]
+        sort_set2 = np.argsort(stat2)
+        pval1 = diff_exp.pvalue[set1][sort_set1]
+        pval2 = diff_exp.pvalue[set2][sort_set2]
+        mean_ex1_set1 = np.mean(expression_1[:, set1], axis=0)[sort_set1]
+        mean_ex2_set1 = np.mean(expression_2[:, set1], axis=0)[sort_set1]
+        mean_ex1_set2 = np.mean(expression_1[:, set2], axis=0)[sort_set2]
+        mean_ex2_set2 = np.mean(expression_2[:, set2], axis=0)[sort_set2]
+        mean_diff1 = mean_ex1_set1 - mean_ex2_set1
+        mean_diff2 = mean_ex1_set2 - mean_ex2_set2
+        genes_cellset_1 = self.data.var_names[set1][sort_set1]
+        genes_cellset_2 = self.data.var_names[set2][sort_set2]
+        return {
+            "celllist1": {
+                "topgenes": genes_cellset_1.tolist()[:num_genes],
+                "mean_expression_cellset1": mean_ex1_set1.tolist()[:num_genes],
+                "mean_expression_cellset2": mean_ex2_set1.tolist()[:num_genes],
+                "pval": pval1.tolist()[:num_genes],
+                "ave_diff": mean_diff1.tolist()[:num_genes]
+            },
+            "celllist2": {
+                "topgenes": genes_cellset_2.tolist()[:num_genes],
+                "mean_expression_cellset1": mean_ex1_set2.tolist()[:num_genes],
+                "mean_expression_cellset2": mean_ex2_set2.tolist()[:num_genes],
+                "pval": pval2.tolist()[:num_genes],
+                "ave_diff": mean_diff2.tolist()[:num_genes]
+            },
+        }
+
+    def expression(self, cells=None, genes=None):
+        """
+        Retrieves expression for each gene for cells in data frame
+        :param df:
+        :return: {
+            "genes": list of genes,
+            "cells": list of cells and expression list,
+            "nonzero_gene_count": number of nonzero genes
+        }
+        """
+        if cells:
+            cells_idx = np.in1d(self.data.obs["cell_name"], cells)
+        else:
+            cells_idx = np.ones((self.cell_count,), dtype=bool)
+        if genes:
+            genes_idx = np.in1d(self.data.var_names, genes)
+        else:
+            genes_idx = np.ones((self.gene_count,), dtype=bool)
+        index = np.ix_(cells_idx, genes_idx)
+        expression = self.data.X[index]
+
+        if not genes:
+            genes = self.data.var.index.tolist()
+        if not cells:
+            cells = self.data.obs["cell_name"].tolist()
+
+        cell_data = []
+        for idx, cell in enumerate(cells):
+            cell_data.append({
+                "cellname": cell,
+                "e": list(expression[idx]),
+            })
+
+        return {
+            "genes": genes,
+            "cells": cell_data,
+            "nonzero_gene_count": int(np.sum(expression.any(axis=0)))
+        }

server/app/util/filter.py

@@ -0,0 +1,70 @@
+class QueryStringError(Exception):
+    pass
+
+
+def _convert_variable(datatype, variable):
+    """
+    Convert variable to number (float/int)
+    Used for dataset metadata and for query string
+    :param datatype: type to convert to
+    :param variable (string or None): value of variable
+    :return: converted variable
+    :raises: ValueError
+    """
+    try:
+        if variable is None:
+            return variable
+        if datatype == "int":
+            variable = int(variable)
+        elif datatype == "float":
+            variable = float(variable)
+        return variable
+    except ValueError:
+        raise
+
+
+def parse_filter(filter, schema):
+    """
+    The filter comes in as arguments from a GET/POST request
+    For categorical metadata keys filter based on key=value
+    For continuous metadata keys filter by key=min,max
+    Either value can be replaced by a * To have only a minimum value key=min, To have only a maximum value key=*,max
+
+    They combine via AND so a cell's metadata would have to match every filter
+
+    The results is a matrix with the cells the pass the filter and at this point all the genes
+    :param filter: flask's request.args
+    :param schema: dictionary schema
+    :return:
+    """
+    query = {}
+    for key in filter:
+        value = filter.getlist(key)
+        if key not in schema:
+            raise QueryStringError("Error: key {} not in metadata schema".format(key))
+        query[key] = {
+            "variable_type": schema[key]["variabletype"],
+            "value_type": schema[key]["type"]
+        }
+        if query[key]["variable_type"] == "categorical":
+            query[key]["query"] = [_convert_variable(query[key]["value_type"], v) for v in value]
+        elif query[key]["variable_type"] == "continuous":
+            value = value[0]
+            try:
+                min, max = value.split(",")
+            except ValueError:
+                raise QueryStringError("Error: min,max format required for range for key {}, got {}".format(key, value))
+            if min == "*":
+                min = None
+            if max == "*":
+                max = None
+            try:
+                query[key]["query"] = {
+                    "min": _convert_variable(query[key]["value_type"], min),
+                    "max": _convert_variable(query[key]["value_type"], max)
+                }
+            except ValueError:
+                raise QueryStringError(
+                    "Error: expected type {} for key {}, got {}".format(query[key]["type"], key, value)
+                )
+    return query

server/app/util/schema_parse.py

@@ -0,0 +1,7 @@
+import json
+
+
+def parse_schema(filename):
+    with open(filename) as fh:
+        schema = json.load(fh)
+    return schema

server/app/util/utils.py

@@ -0,0 +1,40 @@
+import json
+
+from numpy import float32, integer
+from flask import make_response, jsonify, Response
+
+
+class Float32JSONEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, float32):
+            return float(obj)
+        elif isinstance(obj, integer):
+            return int(obj)
+        return json.JSONEncoder.default(self, obj)
+
+
+def make_payload(data, errormessage="", errorcode=200):
+    """
+    Creates JSON respons for requests
+    :param data: json data
+    :param errormessage: error message
+    :param errorcode: http error code
+    :return: flask json repsonse
+    """
+    error = False
+    if errormessage:
+        error = True
+    # Questionable
+    data = json.loads(json.dumps(data, cls=Float32JSONEncoder))
+    return make_response(jsonify({
+        "data": data,
+        "status": {
+            "error": error,
+            "errormessage": errormessage,
+        }
+    }), errorcode)
+
+
+def make_streaming_response(data_generator, errorcode=200, content_type="application/json"):
+    # TODO headers
+    return Response(data_generator, status=errorcode, content_type=content_type)

server/app/web/templates/swagger.html

@@ -0,0 +1,97 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+	<meta charset="UTF-8">
+	<title>CellxGene REST API - Swagger definition</title>
+	<link href="https://fonts.googleapis.com/css?family=Open+Sans:400,700|Source+Code+Pro:300,600|Titillium+Web:400,600,700"
+		  rel="stylesheet">
+	<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.2.1/swagger-ui.css"
+		  crossorigin="anonymous"/>
+	<style>
+		html {
+			box-sizing: border-box;
+			overflow: -moz-scrollbars-vertical;
+			overflow-y: scroll;
+		}
+
+		*,
+		*:before,
+		*:after {
+			box-sizing: inherit;
+		}
+
+		body {
+			margin: 0;
+			background: #fafafa;
+		}
+	</style>
+</head>
+
+<body>
+
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"
+	 style="position:absolute;width:0;height:0">
+	<defs>
+		<symbol viewBox="0 0 20 20" id="unlocked">
+			<path d="M15.8 8H14V5.6C14 2.703 12.665 1 10 1 7.334 1 6 2.703 6 5.6V6h2v-.801C8 3.754 8.797 3 10 3c1.203 0 2 .754 2 2.199V8H4c-.553 0-1 .646-1 1.199V17c0 .549.428 1.139.951 1.307l1.197.387C5.672 18.861 6.55 19 7.1 19h5.8c.549 0 1.428-.139 1.951-.307l1.196-.387c.524-.167.953-.757.953-1.306V9.199C17 8.646 16.352 8 15.8 8z"></path>
+		</symbol>
+
+		<symbol viewBox="0 0 20 20" id="locked">
+			<path d="M15.8 8H14V5.6C14 2.703 12.665 1 10 1 7.334 1 6 2.703 6 5.6V8H4c-.553 0-1 .646-1 1.199V17c0 .549.428 1.139.951 1.307l1.197.387C5.672 18.861 6.55 19 7.1 19h5.8c.549 0 1.428-.139 1.951-.307l1.196-.387c.524-.167.953-.757.953-1.306V9.199C17 8.646 16.352 8 15.8 8zM12 8H8V5.199C8 3.754 8.797 3 10 3c1.203 0 2 .754 2 2.199V8z"/>
+		</symbol>
+
+		<symbol viewBox="0 0 20 20" id="close">
+			<path d="M14.348 14.849c-.469.469-1.229.469-1.697 0L10 11.819l-2.651 3.029c-.469.469-1.229.469-1.697 0-.469-.469-.469-1.229 0-1.697l2.758-3.15-2.759-3.152c-.469-.469-.469-1.228 0-1.697.469-.469 1.228-.469 1.697 0L10 8.183l2.651-3.031c.469-.469 1.228-.469 1.697 0 .469.469.469 1.229 0 1.697l-2.758 3.152 2.758 3.15c.469.469.469 1.229 0 1.698z"/>
+		</symbol>
+
+		<symbol viewBox="0 0 20 20" id="large-arrow">
+			<path d="M13.25 10L6.109 2.58c-.268-.27-.268-.707 0-.979.268-.27.701-.27.969 0l7.83 7.908c.268.271.268.709 0 .979l-7.83 7.908c-.268.271-.701.27-.969 0-.268-.269-.268-.707 0-.979L13.25 10z"/>
+		</symbol>
+
+		<symbol viewBox="0 0 20 20" id="large-arrow-down">
+			<path d="M17.418 6.109c.272-.268.709-.268.979 0s.271.701 0 .969l-7.908 7.83c-.27.268-.707.268-.979 0l-7.908-7.83c-.27-.268-.27-.701 0-.969.271-.268.709-.268.979 0L10 13.25l7.418-7.141z"/>
+		</symbol>
+
+
+		<symbol viewBox="0 0 24 24" id="jump-to">
+			<path d="M19 7v4H5.83l3.58-3.59L8 6l-6 6 6 6 1.41-1.41L5.83 13H21V7z"/>
+		</symbol>
+
+		<symbol viewBox="0 0 24 24" id="expand">
+			<path d="M10 18h4v-2h-4v2zM3 6v2h18V6H3zm3 7h12v-2H6v2z"/>
+		</symbol>
+
+	</defs>
+</svg>
+
+<div id="swagger-ui"></div>
+
+<script src="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.2.1/swagger-ui-bundle.js"
+		 crossorigin="anonymous"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.2.1/swagger-ui-standalone-preset.js"
+		crossorigin="anonymous"></script>
+<script>
+	window.onload = function () {
+		const ui = SwaggerUIBundle({
+			url: window.location.origin + "/api/swagger.json",
+			dom_id: '#swagger-ui',
+			deepLinking: true,
+			presets: [
+				SwaggerUIBundle.presets.apis,
+				SwaggerUIStandalonePreset
+			],
+			plugins: [
+				SwaggerUIBundle.plugins.DownloadUrl
+			],
+			layout: "StandaloneLayout"
+		});
+
+		window.ui = ui
+	}
+</script>
+</body>
+
+</html>
+
+
+

server/app/web/webapp.py

@@ -0,0 +1,24 @@
+from flask import (
+    Blueprint, render_template, url_for, current_app
+)
+
+bp = Blueprint("webapp", __name__, template_folder="templates")
+
+
+@bp.route("/")
+def index():
+    url_base = current_app.config["CXG_API_BASE"]
+    dataset_title = current_app.config["DATASET_TITLE"]
+    return render_template("index.html", prefix=url_base, datasetTitle=dataset_title)
+
+
+# renders swagger documentation
+@bp.route("/swagger")
+def swag():
+    return render_template("swagger.html")
+
+
+# renders swagger documentation
+@bp.route("/favicon.png")
+def favicon():
+    return url_for("static", filename="img/favicon.png")

server/requirements.txt

@@ -0,0 +1,41 @@
+aniso8601==3.0.2
+anndata==0.6.1
+certifi==2018.4.16
+chardet==3.0.4
+click==6.7
+cycler==0.10.0
+decorator==4.3.0
+Flask==0.12.4
+Flask-Compress==1.4.0
+Flask-Cors==3.0.6
+Flask-RESTful==0.3.6
+flask-restful-swagger-2==0.35
+h5py==2.8.0
+idna==2.7
+itsdangerous==0.24
+Jinja2==2.10
+joblib==0.12.0
+kiwisolver==1.0.1
+llvmlite==0.23.2
+MarkupSafe==1.0
+matplotlib==2.2.2
+natsort==5.3.2
+networkx==2.1
+numba==0.38.1
+numexpr==2.6.5
+numpy==1.14.5
+pandas==0.23.1
+patsy==0.5.0
+pyparsing==2.2.0
+python-dateutil==2.7.3
+pytz==2018.4
+requests==2.19.1
+scanpy==1.2.2
+scikit-learn==0.19.1
+scipy==1.1.0
+seaborn==0.8.1
+six==1.11.0
+statsmodels==0.9.0
+tables==3.4.4
+urllib3==1.23
+Werkzeug==0.14.1