bumpversion to 0.7.0 (#623)

* initial dataframe commit

* initial dataframe port of core app

* rename variables for clarity

* remove unused import

* comment out unused code

* fix array handling bug in crossfilter dimension creation

* allow creation of empty dataframes

* handle non-existent columns

* handle non-existent columns

* revise tests for new dataframe

* comments for clarity

* comments for clarity

* generate bulk add placeholder with real gene names

* fix bug in gene name adding

* more dataframe unit tests

* fix bug - subset from current world, not universe

* put cut and pasted code into a single function

* improve caching of crossfilter

* remove cascading update bug from graph

* more performance work

* improve state handling for scatterplot

* performance optimization of critical path

* add column summarization

* dataframe utils

* add callOnceLazy

* fix tests

* minor updates found during review

* fix misspelling

* remove RESTv02 from function names

* comment cleanup

* cut/icut col parameter defaults to null

* break up large test

* improve tests and comments on dataframe at/has functions

* add Dataframe withCol/dropCol

* expression varData now stored in a dataframe

* dead code cleanup

* use dataframe.summarize()

* test cases for Dataframe.col.summarize

* update test cases for new dataframe summarize

* improve naming

* use new hasCol API

* add comments

* add more Dataframe.withCol tests

* add ability to specify row index in cut operation

* retire subsetVarData function

* correctly handle expression subsetting

* lint and improve comments

* rename cut to subset

* create helper file for controls reducer

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.6.1
+current_version = 0.7.0
 
 [bumpversion:file:setup.py]
 search = version="{current_version}"

.travis.yml

@@ -12,10 +12,18 @@ install:
   - make install
   - pip install -r server/requirements-dev.txt
   - docker build .
-script:
-  - set -eo pipefail
-  - flake8 server
-  - black --check
-  - npm run --prefix client/ build
-  - npm run --prefix client/ test
-  - pytest -s server/test
+
+jobs:
+  include:
+  - name: "Branch Tests"
+    script:
+      - set -eo pipefail
+      - flake8 server
+      - black --check
+      - npm run --prefix client/ build
+      - npm run --prefix client/ unit-test
+      - pytest -s server/test
+  - name: "Smoke Tests"
+    if: branch = master AND type = cron
+    script:
+      - npm run --prefix client/ smoke-test

client/__tests__/e2e/e2e.test.js

@@ -0,0 +1,185 @@
+import puppeteer from "puppeteer";
+
+const jest_env = process.env.JEST_ENV || "dev";
+const appPort = process.env.JEST_CXG_PORT || 3000;
+const appUrlBase = `http://localhost:${appPort}`;
+const DEV = jest_env === "dev";
+
+let browser;
+let page;
+const browserViewport = { width: 1280, height: 960 };
+
+beforeAll(async () => {
+  const browser_params = DEV
+    ? { headless: false, slowMo: 100, devtools: true }
+    : {};
+  browser = await puppeteer.launch(browser_params);
+  page = await browser.newPage();
+  page.setViewport(browserViewport);
+  if (DEV) page.on("console", msg => console.log("PAGE LOG:", msg.text()));
+});
+
+afterAll(() => {
+  if (!DEV) {
+    browser.close();
+  }
+});
+
+const getOneElementInnerHTML = async function(selector) {
+  let text = await page.$eval(selector, el => el.innerHTML);
+  return text;
+};
+
+const drag = async function(el_box, start, end, lasso = false) {
+  const x1 = el_box.content[0].x + start.x;
+  const x2 = el_box.content[0].x + end.x;
+  const y1 = el_box.content[0].y + start.y;
+  const y2 = el_box.content[0].y + end.y;
+  await page.mouse.move(x1, y1);
+  await page.mouse.down();
+  if (lasso) {
+    await page.mouse.move(x2, y1);
+    await page.mouse.move(x2, y2);
+    await page.mouse.move(x1, y2);
+    await page.mouse.move(x1, y1);
+  } else {
+    await page.mouse.move(x2, y2);
+  }
+  await page.mouse.up();
+};
+
+describe("did launch", () => {
+  test("page launched", async () => {
+    await page.goto(appUrlBase);
+    let el = await getOneElementInnerHTML("[data-testid='header']");
+    expect(el).toBe("cellxgene: pbmc3k");
+  });
+});
+
+describe("search for genes", () => {
+  test("search for known gene and add to metadata", async () => {
+    await page.goto(appUrlBase);
+    await page.waitForSelector("[ data-testid='gene-search']");
+    // blueprint's  typeahead is treating typing weird, clicking & waiting first solves this
+    await page.click("[data-testid='gene-search']");
+    await page.waitFor(200);
+    await page.type("[data-testid='gene-search']", "ACD");
+    await page.keyboard.press("Enter");
+    await page.waitForSelector("[data-testid='histogram-ACD']");
+  });
+});
+
+describe("select cells and diffexp", () => {
+  test("selects cells from layout and adds to cell set 1", async () => {
+    await page.goto(appUrlBase);
+    const layout = await page.waitForSelector("[data-testid='layout']");
+    const size = await layout.boxModel();
+    const cellset1 = {
+      start: {
+        x: Math.floor(size.width * 0.25),
+        y: Math.floor(size.height * 0.25)
+      },
+      end: {
+        x: Math.floor(size.width * 0.35),
+        y: Math.floor(size.height * 0.35)
+      }
+    };
+    await drag(size, cellset1.start, cellset1.end, true);
+    await page.click("[data-testid='cellset-button-1");
+    let button = await getOneElementInnerHTML("[data-testid='cellset-button-1");
+    expect(button).toMatch(/26 cells/);
+  });
+
+  test("selects cells from layout and adds to cell set 2", async () => {
+    await page.goto(appUrlBase);
+    const layout = await page.waitForSelector("[data-testid='layout']");
+    const size = await layout.boxModel();
+    const cellset2 = {
+      start: {
+        x: Math.floor(size.width * 0.45),
+        y: Math.floor(size.height * 0.45)
+      },
+      end: {
+        x: Math.floor(size.width * 0.55),
+        y: Math.floor(size.height * 0.55)
+      }
+    };
+    await drag(size, cellset2.start, cellset2.end, true);
+    await page.click("[data-testid='cellset-button-2");
+    let button = await getOneElementInnerHTML("[data-testid='cellset-button-2");
+    expect(button).toMatch(/49 cells/);
+  });
+
+  test("selects cells, saves them and performs diffexp", async () => {
+    await page.goto(appUrlBase);
+    const layout = await page.waitForSelector("[data-testid='layout']");
+    const size = await layout.boxModel();
+    const cellset1 = {
+      start: {
+        x: Math.floor(size.width * 0.25),
+        y: Math.floor(size.height * 0.25)
+      },
+      end: {
+        x: Math.floor(size.width * 0.35),
+        y: Math.floor(size.height * 0.35)
+      }
+    };
+    await drag(size, cellset1.start, cellset1.end, true);
+    await page.click("[data-testid='cellset-button-1");
+    const cellset2 = {
+      start: {
+        x: Math.floor(size.width * 0.45),
+        y: Math.floor(size.height * 0.45)
+      },
+      end: {
+        x: Math.floor(size.width * 0.55),
+        y: Math.floor(size.height * 0.55)
+      }
+    };
+    await drag(size, cellset2.start, cellset2.end, true);
+    await page.click("[data-testid='cellset-button-2");
+    await page.click("[data-testid='diffexp-button");
+    await page.waitForSelector("[data-testclass='histogram-diffexp']");
+    const diffexps = await page.$$eval(
+      "[data-testclass='histogram-diffexp']",
+      divs => {
+        return divs.map(div =>
+          div.id.substring("histogram-".length, div.id.length)
+        );
+      }
+    );
+    expect(diffexps).toMatchObject([
+      "HLA-DPA1",
+      "HLA-DQA1",
+      "HLA-DRB1",
+      "HLA-DMA",
+      "CST3",
+      "HLA-DPB1",
+      "HLA-DQB1",
+      "LGALS2",
+      "FCER1A",
+      "LTB"
+    ]);
+  });
+});
+
+describe("brushable histogram", () => {
+  test("can brush historgram", async () => {
+    await page.goto(appUrlBase);
+    const hist = await page.waitForSelector(
+      "[data-testid='histogram_n_genes_svg-brush'] > .overlay"
+    );
+    const hist_size = await hist.boxModel();
+    const draghist = {
+      start: {
+        x: Math.floor(hist_size.width * 0.25),
+        y: Math.floor(hist_size.height * 0.5)
+      },
+      end: {
+        x: Math.floor(hist_size.width * 0.55),
+        y: Math.floor(hist_size.height * 0.5)
+      }
+    };
+    await drag(hist_size, draghist.start, draghist.end);
+  });
+});

client/__tests__/util/dataframe/dataframe.test.js

@@ -0,0 +1,616 @@
+import * as Dataframe from "../../../src/util/dataframe";
+
+describe("dataframe constructor", () => {
+  test("empty dataframe", () => {
+    const df = new Dataframe.Dataframe([0, 0], []);
+    expect(df).toBeDefined();
+    expect(df.dims).toEqual([0, 0]);
+    expect(df).toHaveLength(0);
+    expect(df.icol(0)).not.toBeDefined();
+  });
+
+  test("create with default indices", () => {
+    const df = new Dataframe.Dataframe(
+      [3, 2],
+      [new Int32Array(3).fill(0), new Int32Array(3).fill(1)]
+    );
+
+    expect(df).toBeDefined();
+    expect(df.dims).toEqual([3, 2]);
+    expect(df.rowIndex).toBeInstanceOf(Dataframe.IdentityInt32Index);
+    expect(df.colIndex).toBeInstanceOf(Dataframe.IdentityInt32Index);
+    expect(df.at(0, 0)).toEqual(0);
+    expect(df.at(2, 1)).toEqual(1);
+    expect(df.iat(0, 0)).toEqual(0);
+    expect(df.iat(2, 1)).toEqual(1);
+  });
+
+  test("create with labelled indices", () => {
+    const df = new Dataframe.Dataframe(
+      [3, 2],
+      [new Int32Array([0, 1, 2]), new Int32Array([3, 4, 5])],
+      new Dataframe.DenseInt32Index([2, 1, 0]),
+      new Dataframe.KeyIndex(["A", "B"])
+    );
+
+    expect(df).toBeDefined();
+    expect(df.dims).toEqual([3, 2]);
+
+    expect(df.rowIndex).toBeInstanceOf(Dataframe.DenseInt32Index);
+    expect(df.colIndex).toBeInstanceOf(Dataframe.KeyIndex);
+    expect(df.rowIndex.keys()).toEqual(new Int32Array([2, 1, 0]));
+    expect(df.colIndex.keys()).toEqual(["A", "B"]);
+
+    expect(df.at(0, "A")).toEqual(2);
+    expect(df.at(2, "B")).toEqual(3);
+    expect(df.iat(0, 0)).toEqual(0);
+    expect(df.iat(2, 1)).toEqual(5);
+  });
+});
+
+describe("simple data access", () => {
+  const df = new Dataframe.Dataframe(
+    [4, 2],
+    [
+      new Float64Array([0.0, Number.NaN, Number.POSITIVE_INFINITY, 3.14159]),
+      ["red", "blue", "green", "nan"]
+    ],
+    new Dataframe.DenseInt32Index([3, 2, 1, 0]),
+    new Dataframe.KeyIndex(["numbers", "colors"])
+  );
+
+  test("iat", () => {
+    expect(df).toBeDefined();
+
+    // present
+    expect(df.iat(0, 0)).toEqual(0.0);
+    expect(df.iat(0, 1)).toEqual("red");
+    expect(df.iat(1, 0)).toEqual(Number.NaN);
+    expect(df.iat(1, 1)).toEqual("blue");
+    expect(df.iat(2, 0)).toEqual(Number.POSITIVE_INFINITY);
+    expect(df.iat(2, 1)).toEqual("green");
+    expect(df.iat(3, 0)).toEqual(3.14159);
+    expect(df.iat(3, 1)).toEqual("nan");
+
+    // labels out of range have no defined behavior
+  });
+
+  test("at", () => {
+    expect(df).toBeDefined();
+
+    // present
+    expect(df.at(3, "numbers")).toEqual(0.0);
+    expect(df.at(3, "colors")).toEqual("red");
+    expect(df.at(2, "numbers")).toEqual(Number.NaN);
+    expect(df.at(2, "colors")).toEqual("blue");
+    expect(df.at(1, "numbers")).toEqual(Number.POSITIVE_INFINITY);
+    expect(df.at(1, "colors")).toEqual("green");
+    expect(df.at(0, "numbers")).toEqual(3.14159);
+    expect(df.at(0, "colors")).toEqual("nan");
+
+    // labels out of range have no defined behavior
+  });
+
+  test("ihas", () => {
+    expect(df).toBeDefined();
+
+    // present
+    expect(df.ihas(0, 0)).toBeTruthy();
+    expect(df.ihas(1, 1)).toBeTruthy();
+    expect(df.ihas(3, 1)).toBeTruthy();
+
+    // not present
+    expect(df.ihas(-1, -1)).toBeFalsy();
+    expect(df.ihas(0, 99)).toBeFalsy();
+    expect(df.ihas(99, 0)).toBeFalsy();
+    expect(df.ihas(99, 99)).toBeFalsy();
+    expect(df.ihas(-1, 0)).toBeFalsy();
+    expect(df.ihas(0, -1)).toBeFalsy();
+  });
+
+  test("has", () => {
+    expect(df).toBeDefined();
+
+    // present
+    expect(df.has(3, "numbers")).toBeTruthy();
+    expect(df.has(0, "numbers")).toBeTruthy();
+    expect(df.has(3, "colors")).toBeTruthy();
+    expect(df.has(0, "colors")).toBeTruthy();
+
+    // not present
+    expect(df.has(3, "foo")).toBeFalsy();
+    expect(df.has(-1, "numbers")).toBeFalsy();
+    expect(df.has(-1, -1)).toBeFalsy();
+    expect(df.has(null, null)).toBeFalsy();
+    expect(df.has(0, "foo")).toBeFalsy();
+    expect(df.has(99, "numbers")).toBeFalsy();
+    expect(df.has(99, "foo")).toBeFalsy();
+  });
+});
+
+describe("dataframe subsetting", () => {
+  describe("subset", () => {
+    const sourceDf = new Dataframe.Dataframe(
+      [3, 4],
+      [
+        new Int32Array([0, 1, 2]),
+        ["A", "B", "C"],
+        new Float32Array([4.4, 5.5, 6.6]),
+        ["red", "green", "blue"]
+      ],
+      null,
+      new Dataframe.KeyIndex(["int32", "string", "float32", "colors"])
+    );
+
+    test("all rows, one column", () => {
+      const dfA = sourceDf.subset(null, ["colors"]);
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([3, 1]);
+      expect(dfA.iat(0, 0)).toEqual("red");
+      expect(dfA.at(2, "colors")).toEqual("blue");
+      expect(dfA.col("colors").asArray()).toEqual(["red", "green", "blue"]);
+      expect(dfA.icol(0).asArray()).toEqual(["red", "green", "blue"]);
+      expect(dfA.col("colors").asArray()).toEqual(
+        sourceDf.col("colors").asArray()
+      );
+      expect(dfA.rowIndex.keys()).toEqual(sourceDf.rowIndex.keys());
+      expect(dfA.colIndex.keys()).toEqual(["colors"]);
+    });
+
+    test("all rows, two columns", () => {
+      const dfB = sourceDf.subset(null, ["colors", "float32"]);
+      expect(dfB).toBeDefined();
+      expect(dfB.dims).toEqual([3, 2]);
+      expect(dfB.iat(0, 0)).toBeCloseTo(4.4);
+      expect(dfB.iat(0, 1)).toEqual("red");
+      expect(dfB.at(2, "colors")).toEqual("blue");
+      expect(dfB.at(2, "float32")).toBeCloseTo(6.6);
+      expect(dfB.col("colors").asArray()).toEqual(["red", "green", "blue"]);
+      expect(dfB.col("float32").asArray()).toEqual(
+        new Float32Array([4.4, 5.5, 6.6])
+      );
+      expect(dfB.icol(0).asArray()).toEqual(dfB.col("float32").asArray());
+      expect(dfB.icol(1).asArray()).toEqual(dfB.col("colors").asArray());
+      expect(dfB.col("colors").asArray()).toEqual(
+        sourceDf.col("colors").asArray()
+      );
+      expect(dfB.col("float32").asArray()).toEqual(
+        sourceDf.col("float32").asArray()
+      );
+      expect(dfB.rowIndex.keys()).toEqual(sourceDf.rowIndex.keys());
+      expect(dfB.colIndex.keys()).toEqual(["float32", "colors"]);
+    });
+
+    test("one row, all columns", () => {
+      const dfC = sourceDf.subset([1], null);
+      expect(dfC).toBeDefined();
+      expect(dfC.dims).toEqual([1, 4]);
+      expect(dfC.iat(0, 0)).toEqual(1);
+      expect(dfC.iat(0, 1)).toEqual("B");
+      expect(dfC.iat(0, 2)).toBeCloseTo(5.5);
+      expect(dfC.iat(0, 3)).toEqual("green");
+      expect(dfC.rowIndex.keys()).toEqual(new Int32Array([1]));
+      expect(dfC.colIndex.keys()).toEqual(sourceDf.colIndex.keys());
+    });
+
+    test("two rows, all columns", () => {
+      const dfD = sourceDf.subset([0, 2], null);
+      expect(dfD).toBeDefined();
+      expect(dfD.dims).toEqual([2, 4]);
+      expect(dfD.icol(0).asArray()).toEqual(new Int32Array([0, 2]));
+      expect(dfD.icol(1).asArray()).toEqual(["A", "C"]);
+      expect(dfD.icol(2).asArray()).toEqual(new Float32Array([4.4, 6.6]));
+      expect(dfD.icol(3).asArray()).toEqual(["red", "blue"]);
+      expect(dfD.rowIndex.keys()).toEqual(new Int32Array([0, 2]));
+      expect(dfD.colIndex.keys()).toEqual(sourceDf.colIndex.keys());
+    });
+
+    test("all rows, all columns", () => {
+      const dfE = sourceDf.subset(null, null);
+      expect(dfE).toBeDefined();
+      expect(dfE.dims).toEqual([3, 4]);
+      expect(dfE.icol(0).asArray()).toEqual(sourceDf.icol(0).asArray());
+      expect(dfE.icol(1).asArray()).toEqual(sourceDf.icol(1).asArray());
+      expect(dfE.icol(2).asArray()).toEqual(sourceDf.icol(2).asArray());
+      expect(dfE.icol(3).asArray()).toEqual(sourceDf.icol(3).asArray());
+      expect(dfE.rowIndex.keys()).toEqual(sourceDf.rowIndex.keys());
+      expect(dfE.colIndex.keys()).toEqual(sourceDf.colIndex.keys());
+    });
+
+    test("two rows, two colums", () => {
+      const dfF = sourceDf.subset([0, 2], ["int32", "float32"]);
+      expect(dfF).toBeDefined();
+      expect(dfF.dims).toEqual([2, 2]);
+      expect(dfF.icol(0).asArray()).toEqual(new Int32Array([0, 2]));
+      expect(dfF.icol(1).asArray()).toEqual(new Float32Array([4.4, 6.6]));
+      expect(dfF.rowIndex.keys()).toEqual(new Int32Array([0, 2]));
+      expect(dfF.colIndex.keys()).toEqual(["int32", "float32"]);
+    });
+
+    test("withRowIndex", () => {
+      const df = sourceDf.subset(
+        null,
+        ["int32", "float32"],
+        new Dataframe.DenseInt32Index([3, 2, 1])
+      );
+      expect(df.colIndex).toBeInstanceOf(Dataframe.KeyIndex);
+      expect(df.rowIndex).toBeInstanceOf(Dataframe.DenseInt32Index);
+      expect(df.at(3, "int32")).toEqual(df.iat(0, 0));
+    });
+
+    test("withRowIndex error checks", () => {
+      expect(() =>
+        sourceDf.subset(null, ["red"], new Dataframe.IdentityInt32Index(1))
+      ).toThrow(RangeError);
+      expect(() =>
+        sourceDf.subset(null, ["red"], new Dataframe.DenseInt32Index([0, 1]))
+      ).toThrow(RangeError);
+      expect(() =>
+        sourceDf.subset(null, ["red"], new Dataframe.KeyIndex([0, 1, 2, 3]))
+      ).toThrow(RangeError);
+    });
+  });
+
+  test("isubsetMask", () => {
+    const sourceDf = new Dataframe.Dataframe(
+      [3, 4],
+      [
+        new Int32Array([0, 1, 2]),
+        ["A", "B", "C"],
+        new Float32Array([4.4, 5.5, 6.6]),
+        ["red", "green", "blue"]
+      ],
+      new Dataframe.DenseInt32Index([2, 4, 6]),
+      new Dataframe.KeyIndex(["int32", "string", "float32", "colors"])
+    );
+
+    const dfA = sourceDf.isubsetMask(
+      new Uint8Array([0, 1, 1]),
+      new Uint8Array([1, 0, 0, 1])
+    );
+    expect(dfA.dims).toEqual([2, 2]);
+    expect(dfA.icol(0).asArray()).toEqual(new Int32Array([1, 2]));
+    expect(dfA.icol(1).asArray()).toEqual(["green", "blue"]);
+    expect(dfA.rowIndex.keys()).toEqual(new Int32Array([4, 6]));
+    expect(dfA.colIndex.keys()).toEqual(["int32", "colors"]);
+  });
+});
+
+describe("dataframe factories", () => {
+  test("create", () => {
+    const df = Dataframe.Dataframe.create(
+      [3, 3],
+      [
+        new Array(3).fill(0),
+        new Int16Array(3).fill(99),
+        new Float64Array(3).fill(1.1)
+      ]
+    );
+
+    expect(df).toBeDefined();
+    expect(df.dims).toEqual([3, 3]);
+    expect(df).toHaveLength(3);
+    expect(df.iat(0, 0)).toEqual(0);
+    expect(df.iat(1, 1)).toEqual(99);
+    expect(df.iat(2, 2)).toBeCloseTo(1.1);
+    expect(df.iat(0, 0)).toEqual(df.at(0, 0));
+    expect(df.iat(1, 1)).toEqual(df.at(1, 1));
+    expect(df.iat(2, 2)).toEqual(df.at(2, 2));
+  });
+
+  test("clone", () => {
+    const dfA = new Dataframe.Dataframe(
+      [3, 2],
+      [new Int32Array([0, 1, 2]), new Int32Array([3, 4, 5])],
+      new Dataframe.DenseInt32Index([2, 1, 0]),
+      new Dataframe.KeyIndex(["A", "B"])
+    );
+
+    const dfB = dfA.clone();
+    expect(dfB).not.toBe(dfA);
+    expect(dfB.dims).toEqual(dfA.dims);
+    expect(dfB).toHaveLength(dfA.length);
+    expect(dfB.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    expect(dfB.colIndex.keys()).toEqual(dfA.colIndex.keys());
+    for (let i = 0, l = dfB.dims[1]; i < l; i += 1) {
+      expect(dfB.icol(i).asArray()).toEqual(dfA.icol(i).asArray());
+    }
+  });
+
+  describe("withCol", () => {
+    test("KeyIndex", () => {
+      const df = new Dataframe.Dataframe(
+        [2, 2],
+        [["red", "blue"], [true, false]],
+        null,
+        new Dataframe.KeyIndex(["colors", "bools"])
+      );
+      const dfA = df.withCol("numbers", [1, 0]);
+
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([2, 3]);
+      expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.icol(1).asArray()).toEqual([true, false]);
+      expect(dfA.icol(2).asArray()).toEqual([1, 0]);
+      expect(dfA.col("numbers").asArray()).toEqual([1, 0]);
+      expect(dfA.colIndex.keys()).toEqual(["colors", "bools", "numbers"]);
+      expect(df.colIndex.keys()).toEqual(["colors", "bools"]);
+      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    });
+
+    test("DenseInt32Index", () => {
+      const df = new Dataframe.Dataframe(
+        [2, 2],
+        [["red", "blue"], [true, false]],
+        null,
+        new Dataframe.DenseInt32Index([74, 75])
+      );
+      const dfA = df.withCol(72, [1, 0]);
+
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([2, 3]);
+      expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.icol(1).asArray()).toEqual([true, false]);
+      expect(dfA.icol(2).asArray()).toEqual([1, 0]);
+      expect(dfA.col(74).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.col(75).asArray()).toEqual([true, false]);
+      expect(dfA.col(72).asArray()).toEqual([1, 0]);
+      expect(dfA.colIndex.keys()).toEqual(new Int32Array([74, 75, 72]));
+      expect(df.colIndex.keys()).toEqual(new Int32Array([74, 75]));
+      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    });
+
+    test("DenseInt32Index promote", () => {
+      const df = new Dataframe.Dataframe(
+        [2, 2],
+        [["red", "blue"], [true, false]],
+        null,
+        new Dataframe.DenseInt32Index([74, 75])
+      );
+      const dfA = df.withCol(999, [1, 0]);
+
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([2, 3]);
+      expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.icol(1).asArray()).toEqual([true, false]);
+      expect(dfA.icol(2).asArray()).toEqual([1, 0]);
+      expect(dfA.col(74).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.col(75).asArray()).toEqual([true, false]);
+      expect(dfA.col(999).asArray()).toEqual([1, 0]);
+      expect(dfA.colIndex.keys()).toEqual(new Int32Array([74, 75, 999]));
+      expect(df.colIndex.keys()).toEqual(new Int32Array([74, 75]));
+      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    });
+
+    test("IdentityInt32Index with last", () => {
+      const df = new Dataframe.Dataframe(
+        [2, 2],
+        [["red", "blue"], [true, false]],
+        null,
+        null
+      );
+      const dfA = df.withCol(2, [1, 0]);
+
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([2, 3]);
+      expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.icol(1).asArray()).toEqual([true, false]);
+      expect(dfA.icol(2).asArray()).toEqual([1, 0]);
+      expect(dfA.col(0).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.col(1).asArray()).toEqual([true, false]);
+      expect(dfA.col(2).asArray()).toEqual([1, 0]);
+      expect(dfA.colIndex.keys()).toEqual(new Int32Array([0, 1, 2]));
+      expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1]));
+      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    });
+
+    test("IdentityInt32Index promote", () => {
+      const df = new Dataframe.Dataframe(
+        [2, 2],
+        [["red", "blue"], [true, false]],
+        null,
+        null
+      );
+      const dfA = df.withCol(99, [1, 0]);
+
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([2, 3]);
+      expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.icol(1).asArray()).toEqual([true, false]);
+      expect(dfA.icol(2).asArray()).toEqual([1, 0]);
+      expect(dfA.col(0).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.col(1).asArray()).toEqual([true, false]);
+      expect(dfA.col(99).asArray()).toEqual([1, 0]);
+      expect(dfA.colIndex.keys()).toEqual(new Int32Array([0, 1, 99]));
+      expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1]));
+      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    });
+
+    describe("handle column dimensions correctly", () => {
+      /*
+      there are two conditions:
+      - empty dataframe - will accept an add of any dimensionality
+      - non-empty dataframe - added column must match row-count dimension
+      */
+      test("empty.withCol", () => {
+        const edf = Dataframe.Dataframe.empty();
+        const df = edf.withCol("foo", [1, 2, 3]);
+
+        expect(edf).toBeDefined();
+        expect(df).toBeDefined();
+        expect(edf).not.toEqual(df);
+        expect(df.dims).toEqual([3, 1]);
+        expect(df.icol(0).asArray()).toEqual([1, 2, 3]);
+      });
+
+      test("withCol dimension check", () => {
+        const dfA = new Dataframe.Dataframe([1, 1], [["a"]]);
+        expect(() => {
+          dfA.withCol(1, []);
+        }).toThrow(RangeError);
+      });
+    });
+  });
+
+  describe("dropCol", () => {
+    test("KeyIndex", () => {
+      const df = new Dataframe.Dataframe(
+        [2, 3],
+        [["red", "blue"], [true, false], [1, 0]],
+        null,
+        new Dataframe.KeyIndex(["colors", "bools", "numbers"])
+      );
+      const dfA = df.dropCol("colors");
+
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([2, 2]);
+      expect(dfA.icol(0).asArray()).toEqual([true, false]);
+      expect(dfA.icol(1).asArray()).toEqual([1, 0]);
+      expect(dfA.col("numbers").asArray()).toEqual([1, 0]);
+      expect(dfA.colIndex.keys()).toEqual(["bools", "numbers"]);
+      expect(df.colIndex.keys()).toEqual(["colors", "bools", "numbers"]);
+      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    });
+
+    test("IdentityInt32Index drop first", () => {
+      const df = new Dataframe.Dataframe(
+        [2, 3],
+        [["red", "blue"], [true, false], [1, 0]],
+        null,
+        null
+      );
+      const dfA = df.dropCol(0);
+
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([2, 2]);
+      expect(dfA.icol(0).asArray()).toEqual([true, false]);
+      expect(dfA.icol(1).asArray()).toEqual([1, 0]);
+      expect(df.col(1).asArray()).toEqual(dfA.col(1).asArray());
+      expect(df.col(2).asArray()).toEqual(dfA.col(2).asArray());
+      expect(dfA.colIndex.keys()).toEqual(new Int32Array([1, 2]));
+      expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1, 2]));
+      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    });
+
+    test("IdentityInt32Index drop last", () => {
+      const df = new Dataframe.Dataframe(
+        [2, 3],
+        [["red", "blue"], [true, false], [1, 0]],
+        null,
+        null
+      );
+      const dfA = df.dropCol(2);
+
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([2, 2]);
+      expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.icol(1).asArray()).toEqual([true, false]);
+      expect(df.col(0).asArray()).toEqual(dfA.col(0).asArray());
+      expect(df.col(1).asArray()).toEqual(dfA.col(1).asArray());
+      expect(dfA.colIndex.keys()).toEqual(new Int32Array([0, 1]));
+      expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1, 2]));
+      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    });
+
+    test("DenseInt32Index", () => {
+      const df = new Dataframe.Dataframe(
+        [2, 3],
+        [["red", "blue"], [true, false], [1, 0]],
+        null,
+        new Dataframe.DenseInt32Index([102, 101, 100])
+      );
+      const dfA = df.dropCol(101);
+
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([2, 2]);
+      expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.icol(1).asArray()).toEqual([1, 0]);
+      expect(dfA.col(100).asArray()).toEqual([1, 0]);
+      expect(dfA.col(102).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.colIndex.keys()).toEqual(new Int32Array([102, 100]));
+      expect(df.colIndex.keys()).toEqual(new Int32Array([102, 101, 100]));
+      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    });
+  });
+});
+
+describe("dataframe col", () => {
+  let df = null;
+  beforeEach(() => {
+    df = new Dataframe.Dataframe(
+      [2, 2],
+      [[true, false], [1, 0]],
+      null,
+      new Dataframe.KeyIndex(["A", "B"])
+    );
+  });
+
+  test("col", () => {
+    expect(df).toBeDefined();
+    expect(df.col("A")).toBe(df.icol(0));
+    expect(df.col("B")).toBe(df.icol(1));
+    expect(df.col("undefined")).toBeUndefined();
+    expect(df.icol("undefined")).toBeUndefined();
+
+    const colA = df.col("A");
+    expect(colA).toBeInstanceOf(Function);
+    expect(colA.asArray).toBeInstanceOf(Function);
+    expect(colA.has).toBeInstanceOf(Function);
+    expect(colA.ihas).toBeInstanceOf(Function);
+    expect(colA.indexOf).toBeInstanceOf(Function);
+    expect(colA.iget).toBeInstanceOf(Function);
+  });
+
+  test("col.asArray", () => {
+    expect(df).toBeDefined();
+    expect(df.col("A").asArray()).toEqual([true, false]);
+    expect(df.icol(0).asArray()).toEqual([true, false]);
+    expect(df.col("B").asArray()).toEqual([1, 0]);
+    expect(df.icol(1).asArray()).toEqual([1, 0]);
+  });
+
+  test("col.has", () => {
+    expect(df).toBeDefined();
+    expect(df.col("A").has(-1)).toBe(false);
+    expect(df.col("A").has(0)).toBe(true);
+    expect(df.col("A").has(1)).toBe(true);
+    expect(df.col("A").has(2)).toBe(false);
+    expect(df.col("B").has(-1)).toBe(false);
+    expect(df.col("B").has(0)).toBe(true);
+    expect(df.col("B").has(1)).toBe(true);
+    expect(df.col("B").has(2)).toBe(false);
+  });
+
+  test("col.ihas", () => {
+    expect(df).toBeDefined();
+    expect(df.col("A").ihas(-1)).toBe(false);
+    expect(df.col("A").ihas(0)).toBe(true);
+    expect(df.col("A").ihas(1)).toBe(true);
+    expect(df.col("A").ihas(2)).toBe(false);
+    expect(df.col("B").ihas(-1)).toBe(false);
+    expect(df.col("B").ihas(0)).toBe(true);
+    expect(df.col("B").ihas(1)).toBe(true);
+    expect(df.col("B").ihas(2)).toBe(false);
+  });
+
+  test("col.iget", () => {
+    expect(df).toBeDefined();
+    expect(df.col("A").iget(0)).toEqual(df.iat(0, 0));
+    expect(df.col("B").iget(1)).toEqual(df.iat(1, 1));
+  });
+
+  test("col.indexOf", () => {
+    expect(df).toBeDefined();
+    expect(df.col("A").indexOf(true)).toEqual(0);
+    expect(df.col("A").indexOf(false)).toEqual(1);
+    expect(df.col("A").indexOf(99)).toBeUndefined();
+    expect(df.col("A").indexOf(undefined)).toBeUndefined();
+    expect(df.col("A").indexOf(1)).toBeUndefined();
+
+    expect(df.col("B").indexOf(1)).toEqual(0);
+    expect(df.col("B").indexOf(0)).toEqual(1);
+    expect(df.col("B").indexOf(99)).toBeUndefined();
+    expect(df.col("B").indexOf(undefined)).toBeUndefined();
+    expect(df.col("B").indexOf(true)).toBeUndefined();
+  });
+});

client/__tests__/util/dataframe/summarize.test.js

@@ -0,0 +1,253 @@
+import * as Dataframe from "../../../src/util/dataframe";
+
+function float32Conversion(f) {
+  return new Float32Array([f])[0];
+}
+
+describe("Dataframe column summary", () => {
+  test("empty column test", () => {
+    const df = Dataframe.Dataframe.create([0, 1], [[]]);
+    const summary = df.icol(0).summarize();
+    expect(summary).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: [],
+        categoryCounts: new Map(),
+        numCategories: 0
+      })
+    );
+  });
+
+  test("simple test", () => {
+    const df = new Dataframe.Dataframe(
+      [1, 6],
+      [
+        ["n1"],
+        ["hi"],
+        [true],
+        new Float32Array([39.3]),
+        new Int32Array([99]),
+        [1]
+      ],
+      null,
+      new Dataframe.KeyIndex([
+        "name",
+        "nameString",
+        "nameBoolean",
+        "nameFloat32",
+        "nameInt32",
+        "nameCategorical"
+      ])
+    );
+
+    expect(df.icol(0).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: ["n1"],
+        categoryCounts: new Map([["n1", 1]]),
+        numCategories: 1
+      })
+    );
+    expect(df.icol(1).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: ["hi"],
+        categoryCounts: new Map([["hi", 1]]),
+        numCategories: 1
+      })
+    );
+    expect(df.icol(2).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: [true],
+        categoryCounts: new Map([[true, 1]]),
+        numCategories: 1
+      })
+    );
+    expect(df.icol(3).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: false,
+        min: float32Conversion(39.3),
+        max: float32Conversion(39.3),
+        nan: 0,
+        ninf: 0,
+        pinf: 0
+      })
+    );
+    expect(df.icol(4).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: false,
+        min: 99,
+        max: 99,
+        nan: 0,
+        ninf: 0,
+        pinf: 0
+      })
+    );
+    expect(df.icol(5).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: [1],
+        categoryCounts: new Map([[1, 1]]),
+        numCategories: 1
+      })
+    );
+  });
+
+  test("multi test", () => {
+    const df = new Dataframe.Dataframe(
+      [3, 6],
+      [
+        ["n0", "n1", "n2"],
+        ["hi", "hi", "bye"],
+        [false, true, true],
+        new Float32Array([39.3, 39.3, 0]),
+        new Int32Array([99, 99, 99]),
+        [1, false, "0"]
+      ],
+      null,
+      new Dataframe.KeyIndex([
+        "name",
+        "nameString",
+        "nameBoolean",
+        "nameFloat32",
+        "nameInt32",
+        "nameCategorical"
+      ])
+    );
+
+    expect(df.icol(0).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining(["n0", "n1", "n2"]),
+        categoryCounts: new Map([["n0", 1], ["n1", 1], ["n2", 1]]),
+        numCategories: 3
+      })
+    );
+    expect(df.icol(1).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining(["hi", "bye"]),
+        categoryCounts: new Map([["hi", 2], ["bye", 1]]),
+        numCategories: 2
+      })
+    );
+    expect(df.icol(2).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining([true, false]),
+        categoryCounts: new Map([[true, 2], [false, 1]]),
+        numCategories: 2
+      })
+    );
+    expect(df.icol(3).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: false,
+        min: 0,
+        max: float32Conversion(39.3),
+        nan: 0,
+        ninf: 0,
+        pinf: 0
+      })
+    );
+    expect(df.icol(4).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: false,
+        min: 99,
+        max: 99,
+        nan: 0,
+        ninf: 0,
+        pinf: 0
+      })
+    );
+    expect(df.icol(5).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining([1, false, "0"]),
+        categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]),
+        numCategories: 3
+      })
+    );
+  });
+
+  test("non-finite numbers", () => {
+    const df = new Dataframe.Dataframe(
+      [4, 6],
+      [
+        ["n0", "n1", "n2", "n2"],
+        ["hi", "hi", "bye", "bye"],
+        [false, true, true, true],
+        new Float32Array([
+          39.3,
+          Number.NEGATIVE_INFINITY,
+          Number.NaN,
+          Number.POSITIVE_INFINITY
+        ]),
+        new Int32Array([99, 99, 99, 99]),
+        [1, false, "0", "0"]
+      ],
+      null,
+      new Dataframe.KeyIndex([
+        "name",
+        "nameString",
+        "nameBoolean",
+        "nameFloat32",
+        "nameInt32",
+        "nameCategorical"
+      ])
+    );
+
+    expect(df.icol(0).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining(["n0", "n1", "n2"]),
+        categoryCounts: new Map([["n0", 1], ["n1", 1], ["n2", 2]]),
+        numCategories: 3
+      })
+    );
+    expect(df.icol(1).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining(["hi", "bye"]),
+        categoryCounts: new Map([["hi", 2], ["bye", 1]]),
+        numCategories: 2
+      })
+    );
+    expect(df.icol(2).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining([true, false]),
+        categoryCounts: new Map([[true, 2], [false, 1]]),
+        numCategories: 2
+      })
+    );
+    expect(df.icol(3).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: false,
+        min: float32Conversion(39.3),
+        max: float32Conversion(39.3),
+        nan: 1,
+        ninf: 1,
+        pinf: 1
+      })
+    );
+    expect(df.icol(4).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: false,
+        min: 99,
+        max: 99,
+        nan: 0,
+        ninf: 0,
+        pinf: 0
+      })
+    );
+    expect(df.icol(5).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining([1, false, "0"]),
+        categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]),
+        numCategories: 3
+      })
+    );
+  });
+});

client/__tests__/util/stateManager/keyvalcache.test.js

@@ -1,249 +0,0 @@
-import _ from "lodash";
-import * as kvCache from "../../../src/util/stateManager/keyvalcache";
-
-/*
-This is PRIVATE to keyvalcache and must be kept in sync with
-any changs ot that module.  Need to Know - to enable error handling test
-*/
-const cachePrivateKey = "__kvcachekey__";
-
-/*
-helper function - promisify setTimeout()
-*/
-function timeout(ms) {
-  return new Promise(resolve => setTimeout(resolve, ms));
-}
-
-describe("kvcache API", () => {
-  /*
-  test the happy path create/set/get API
-  */
-
-  test("simple create", () => {
-    /* with defaults */
-    const kvc = kvCache.create();
-    expect(kvc).toBeDefined();
-    expect(kvc).toEqual(expect.objectContaining({}));
-    expect(kvCache.get(kvc, "test")).toBeUndefined();
-
-    /* with params */
-    const kvc1 = kvCache.create(/* lowWatermark */ 99, /* minTTL */ 0);
-    expect(kvc1).toBeDefined();
-    expect(kvc1).toEqual(expect.objectContaining({}));
-  });
-
-  test("set/get", () => {
-    /*
-    - check basic get/set functionality
-    - check set does not mutate source cache
-    */
-    const keyName = "foo";
-    const kvc1 = kvCache.create();
-    expect(kvc1).toBeDefined();
-    expect(kvCache.get(kvc1, keyName)).toBeUndefined();
-
-    const val2 = [2];
-    const kvc2 = kvCache.set(kvc1, keyName, val2);
-    expect(kvc2).toBeDefined();
-    expect(kvc2).not.toBe(kvc1);
-    expect(kvCache.get(kvc1, keyName)).toBeUndefined();
-    expect(kvCache.get(kvc2, keyName)).toBe(val2);
-
-    const val3 = [3];
-    const kvc3 = kvCache.set(kvc2, keyName, val3);
-    expect(kvc3).toBeDefined();
-    expect(kvc3).not.toBe(kvc1);
-    expect(kvc3).not.toBe(kvc2);
-    expect(kvCache.get(kvc1, keyName)).toBeUndefined();
-    expect(kvCache.get(kvc2, keyName)).toBe(val2);
-    expect(kvCache.get(kvc3, keyName)).toBe(val3);
-  });
-});
-
-describe("common error handling", () => {
-  /*
-  Test common error handlers
-  */
-
-  test("set() protection from namespace pollution", () => {
-    /*
-    Test that set() will not allow use of the private cache key
-    */
-    const kvc = kvCache.create();
-    expect(() => {
-      kvCache.set(kvc, cachePrivateKey, {});
-    }).toThrow();
-  });
-
-  test("create() does not accept bogus config", () => {
-    expect(() => {
-      kvCache.create([], {});
-    }).toThrow();
-    expect(() => {
-      kvCache.create(-99, 0);
-    }).toThrow();
-    expect(() => {
-      kvCache.create(100, -1);
-    }).toThrow();
-    expect(() => {
-      kvCache.create(1000, "foobar");
-    }).toThrow();
-    expect(() => {
-      kvCache.create(null, 8);
-    }).toThrow();
-  });
-});
-
-describe("map", () => {
-  /*
-  Test kvCache.map() - create new cache that is a transformation of an
-  existing cache
-  */
-  test("map of empty cache", () => {
-    const kvc = kvCache.create();
-    const callback = jest.fn();
-    const kvcMapped = kvCache.map(kvc, callback);
-    expect(callback).not.toHaveBeenCalled();
-    expect(kvcMapped).toBeDefined();
-    expect(kvcMapped).not.toBe(kvc); // immutable operation
-    expect(kvcMapped).toEqual(kvc);
-  });
-
-  test("map of non-empty cache", () => {
-    const key = "aKey";
-    const val = [0, 1, 2];
-    let kvc = kvCache.create();
-    kvc = kvCache.set(kvc, key, val);
-    const mockCB = jest.fn().mockImplementation(v => [...v]);
-    const kvcMapped = kvCache.map(kvc, mockCB);
-
-    expect(kvcMapped).toBeDefined();
-    expect(kvcMapped).not.toBe(kvc); // immutable operation
-    expect(_.isEqual(kvc, kvcMapped)).toBe(true);
-
-    expect(mockCB).toHaveBeenCalledTimes(1);
-    expect(mockCB).toHaveBeenLastCalledWith(val, key);
-  });
-});
-
-describe("flush", () => {
-  /*
-  test various cache flush behavior
-  */
-  test("flush - lowWatermark, disable minTTL", () => {
-    /*
-    verify lowWatermark functions correctly
-    */
-
-    // set lowWatermark to 2, set three times - only the final two
-    // should remain.
-    let kvc = kvCache.create(2, 0);
-    ["a", "b", "c"].forEach(k => {
-      kvc = kvCache.set(kvc, k, []);
-    });
-
-    expect(kvc).toEqual(
-      expect.objectContaining({
-        b: expect.arrayContaining([]),
-        c: expect.arrayContaining([])
-      })
-    );
-    expect(kvc).toEqual(
-      expect.not.objectContaining({
-        a: expect.arrayContaining([])
-      })
-    );
-  });
-
-  test("flush - minTTL, disable lowWatermark", async () => {
-    /*
-    verify minTTL functions correctly
-    */
-
-    // set minTTL to 1 ms
-    let kvc = kvCache.create(0, 10);
-    kvc = kvCache.set(kvc, "a", []);
-    await timeout(20);
-    ["b", "c"].forEach(k => {
-      kvc = kvCache.set(kvc, k, []);
-    });
-
-    expect(kvc).toEqual(
-      expect.objectContaining({
-        b: expect.arrayContaining([]),
-        c: expect.arrayContaining([])
-      })
-    );
-    expect(kvc).toEqual(
-      expect.not.objectContaining({
-        a: expect.arrayContaining([])
-      })
-    );
-  });
-
-  test("flush - minTTL and lowWatermark", async () => {
-    /*
-    verify minTTL functions correctly
-    */
-
-    // set lowwatermark to 3, minTTL to 1 ms
-    let kvc = kvCache.create(3, 10);
-    kvc = kvCache.set(kvc, "a", []);
-    // delay
-    await timeout(20);
-    ["b", "c"].forEach(k => {
-      kvc = kvCache.set(kvc, k, []);
-    });
-
-    expect(kvc).toEqual(
-      expect.objectContaining({
-        a: expect.arrayContaining([]),
-        b: expect.arrayContaining([]),
-        c: expect.arrayContaining([])
-      })
-    );
-
-    kvc = kvCache.set(kvc, "d", []);
-    expect(kvc).toEqual(
-      expect.objectContaining({
-        b: expect.arrayContaining([]),
-        c: expect.arrayContaining([]),
-        d: expect.arrayContaining([])
-      })
-    );
-    expect(kvc).toEqual(
-      expect.not.objectContaining({
-        a: expect.arrayContaining([])
-      })
-    );
-  });
-
-  test("manual flush", async () => {
-    let kvc = kvCache.create(1, 10);
-    ["a", "b", "c", "d"].forEach(k => {
-      kvc = kvCache.set(kvc, k, []);
-    });
-
-    // Before TTL has expired, should have all values in cache.
-    expect(kvc).toEqual(
-      expect.objectContaining({
-        a: expect.arrayContaining([]),
-        b: expect.arrayContaining([]),
-        c: expect.arrayContaining([])
-      })
-    );
-
-    // let TTL expire
-    await timeout(10);
-
-    // manually flush
-    const postFlushKvc = kvCache.flush(kvc);
-    expect(postFlushKvc).toBeDefined();
-    expect(postFlushKvc).not.toBe(kvc);
-    expect(postFlushKvc).toEqual(
-      expect.objectContaining({
-        d: expect.arrayContaining([])
-      })
-    );
-  });
-});

client/__tests__/util/stateManager/sampleResponses.js

@@ -157,16 +157,6 @@ const anAnnotationsVarFBSResponse = (() => {
   return encodeMatrix(columns, anAnnotationsVarJSONResponse.names);
 })();
 
-const aLayoutJSONResponse = {
-  layout: {
-    ndims: 2,
-    coordinates: _()
-      .range(nObs)
-      .map(idx => [idx, Math.random(), Math.random()])
-      .value()
-  }
-};
-
 const aLayoutFBSResponse = (() => {
   const coords = [
     new Float32Array(nObs).fill(Math.random()),
@@ -190,7 +180,7 @@ const aLayoutFBSResponse = (() => {
 
   NetEncoding.Matrix.startMatrix(builder);
   NetEncoding.Matrix.addNRows(builder, nObs);
-  NetEncoding.Matrix.addNCols(builder, nVar);
+  NetEncoding.Matrix.addNCols(builder, coords.length);
   NetEncoding.Matrix.addColumns(builder, columns);
   const matrix = NetEncoding.Matrix.endMatrix(builder);
   builder.finish(matrix);

client/__tests__/util/stateManager/summarizeAnnotations.test.js

@@ -1,280 +0,0 @@
-import summarizeAnnotations from "../../../src/util/stateManager/summarizeAnnotations";
-
-describe("summarizeAnnotations", () => {
-  const schema = {
-    annotations: {
-      obs: [
-        { name: "name", type: "string" },
-        { name: "nameString", type: "string" },
-        { name: "nameBoolean", type: "boolean" },
-        { name: "nameFloat32", type: "float32" },
-        { name: "nameInt32", type: "int32" },
-        {
-          name: "nameCategorical",
-          type: "categorical",
-          categories: [true, false, 1, 0, 0.00001, 4383.4833, "test", "", "0"]
-        }
-      ],
-      var: [{ name: "name", type: "string" }]
-    }
-  };
-
-  test("empty test", () => {
-    const summary = summarizeAnnotations(schema, [], []);
-    expect(summary).toEqual(
-      expect.objectContaining({
-        obs: {
-          nameString: {
-            categorical: true,
-            categories: [],
-            categoryCounts: new Map(),
-            numCategories: 0
-          },
-          nameBoolean: {
-            categorical: true,
-            categories: [],
-            categoryCounts: new Map(),
-            numCategories: 0
-          },
-          nameFloat32: {
-            categorical: false,
-            range: {
-              max: undefined,
-              min: undefined,
-              nan: 0,
-              ninf: 0,
-              pinf: 0
-            }
-          },
-          nameInt32: {
-            categorical: false,
-            range: {
-              max: undefined,
-              min: undefined,
-              nan: 0,
-              ninf: 0,
-              pinf: 0
-            }
-          },
-          nameCategorical: {
-            categorical: true,
-            categories: [],
-            categoryCounts: new Map(),
-            numCategories: 0
-          }
-        },
-        var: {}
-      })
-    );
-  });
-
-  test("simple test", () => {
-    const obsAnnotations = [
-      {
-        __index__: 0,
-        name: "n1",
-        nameString: "hi",
-        nameBoolean: true,
-        nameFloat32: 39.3,
-        nameInt32: 99,
-        nameCategorical: 1
-      }
-    ];
-    const varAnnotations = [];
-
-    const summary = summarizeAnnotations(
-      schema,
-      obsAnnotations,
-      varAnnotations
-    );
-
-    expect(summary).toEqual(
-      expect.objectContaining({
-        obs: {
-          nameString: {
-            categorical: true,
-            categories: ["hi"],
-            categoryCounts: new Map([["hi", 1]]),
-            numCategories: 1
-          },
-          nameBoolean: {
-            categorical: true,
-            categories: [true],
-            categoryCounts: new Map([[true, 1]]),
-            numCategories: 1
-          },
-          nameFloat32: {
-            categorical: false,
-            range: { min: 39.3, max: 39.3, nan: 0, ninf: 0, pinf: 0 }
-          },
-          nameInt32: {
-            categorical: false,
-            range: { min: 99, max: 99, nan: 0, ninf: 0, pinf: 0 }
-          },
-          nameCategorical: {
-            categorical: true,
-            categories: [1],
-            categoryCounts: new Map([[1, 1]]),
-            numCategories: 1
-          }
-        },
-        var: {}
-      })
-    );
-  });
-
-  test("multi test", () => {
-    const obsAnnotations = [
-      {
-        __index__: 0,
-        name: "n0",
-        nameString: "hi",
-        nameBoolean: false,
-        nameFloat32: 39.3,
-        nameInt32: 99,
-        nameCategorical: 1
-      },
-      {
-        __index__: 1,
-        name: "n1",
-        nameString: "hi",
-        nameBoolean: true,
-        nameFloat32: 39.3,
-        nameInt32: 99,
-        nameCategorical: false
-      },
-      {
-        __index__: 2,
-        name: "n2",
-        nameString: "bye",
-        nameBoolean: true,
-        nameFloat32: 0,
-        nameInt32: 99,
-        nameCategorical: "0"
-      }
-    ];
-    const varAnnotations = [];
-
-    const summary = summarizeAnnotations(
-      schema,
-      obsAnnotations,
-      varAnnotations
-    );
-
-    expect(summary).toMatchObject(
-      expect.objectContaining({
-        obs: {
-          nameString: {
-            categorical: true,
-            categories: expect.arrayContaining(["hi", "bye"]),
-            categoryCounts: new Map([["hi", 2], ["bye", 1]]),
-            numCategories: 2
-          },
-          nameBoolean: {
-            categorical: true,
-            categories: expect.arrayContaining([true, false]),
-            categoryCounts: new Map([[true, 2], [false, 1]]),
-            numCategories: 2
-          },
-          nameFloat32: {
-            categorical: false,
-            range: { min: 0, max: 39.3, nan: 0, ninf: 0, pinf: 0 }
-          },
-          nameInt32: {
-            categorical: false,
-            range: { min: 99, max: 99, nan: 0, ninf: 0, pinf: 0 }
-          },
-          nameCategorical: {
-            categorical: true,
-            categories: expect.arrayContaining([1, false, "0"]),
-            categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]),
-            numCategories: 3
-          }
-        },
-        var: {}
-      })
-    );
-  });
-
-  test("non-finite numbers", () => {
-    const obsAnnotations = [
-      {
-        __index__: 0,
-        name: "n0",
-        nameString: "hi",
-        nameBoolean: false,
-        nameFloat32: 39.3,
-        nameInt32: 99,
-        nameCategorical: 1
-      },
-      {
-        __index__: 1,
-        name: "n1",
-        nameString: "hi",
-        nameBoolean: true,
-        nameFloat32: Number.NEGATIVE_INFINITY,
-        nameInt32: 99,
-        nameCategorical: false
-      },
-      {
-        __index__: 2,
-        name: "n2",
-        nameString: "bye",
-        nameBoolean: true,
-        nameFloat32: Number.NaN,
-        nameInt32: 99,
-        nameCategorical: "0"
-      },
-      {
-        __index__: 3,
-        name: "n2",
-        nameString: "bye",
-        nameBoolean: true,
-        nameFloat32: Number.POSITIVE_INFINITY,
-        nameInt32: 99,
-        nameCategorical: "0"
-      }
-    ];
-    const varAnnotations = [];
-
-    const summary = summarizeAnnotations(
-      schema,
-      obsAnnotations,
-      varAnnotations
-    );
-
-    expect(summary).toMatchObject(
-      expect.objectContaining({
-        obs: {
-          nameString: {
-            categorical: true,
-            categories: expect.arrayContaining(["hi", "bye"]),
-            categoryCounts: new Map([["hi", 2], ["bye", 1]]),
-            numCategories: 2
-          },
-          nameBoolean: {
-            categorical: true,
-            categories: expect.arrayContaining([true, false]),
-            categoryCounts: new Map([[true, 2], [false, 1]]),
-            numCategories: 2
-          },
-          nameFloat32: {
-            categorical: false,
-            range: { min: 39.3, max: 39.3, nan: 1, ninf: 1, pinf: 1 }
-          },
-          nameInt32: {
-            categorical: false,
-            range: { min: 99, max: 99, nan: 0, ninf: 0, pinf: 0 }
-          },
-          nameCategorical: {
-            categorical: true,
-            categories: expect.arrayContaining([1, false, "0"]),
-            categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]),
-            numCategories: 3
-          }
-        },
-        var: {}
-      })
-    );
-  });
-});

client/__tests__/util/stateManager/universe.test.js

@@ -1,13 +1,13 @@
-import _ from "lodash";
 import * as Universe from "../../../src/util/stateManager/universe";
+import * as Dataframe from "../../../src/util/dataframe";
 import * as REST from "./sampleResponses";
 
-describe("createUniverseFromRestV02Response", () => {
+describe("createUniverseFromResponse", () => {
   /*
-  test createUniverseFromRestV02Response - this function converts
+  test createUniverseFromResponse - this function converts
   a set of REST 0.2 responses into a "new" Universe.
 
-  createUniverseFromRestV02Response(
+  createUniverseFromResponse(
     configResponse,
     schemaResponse,
     annotationsObsResponse,
@@ -30,7 +30,7 @@ describe("createUniverseFromRestV02Response", () => {
     create a universe from sample data nad validate its shape & contents
     */
     const { nObs, nVar } = REST.schema.schema.dataframe;
-    const universe = Universe.createUniverseFromRestV02Response(
+    const universe = Universe.createUniverseFromResponse(
       REST.config,
       REST.schema,
       REST.annotationsObs,
@@ -41,27 +41,26 @@ describe("createUniverseFromRestV02Response", () => {
     expect(universe).toBeDefined();
     expect(universe).toMatchObject(
       expect.objectContaining({
-        api: "0.2",
         nObs,
         nVar,
         schema: REST.schema.schema,
-        obsAnnotations: expect.any(Array),
-        varAnnotations: expect.any(Array),
-        obsNameToIndexMap: expect.any(Object),
-        varNameToIndexMap: expect.any(Object),
-        obsLayout: expect.objectContaining({
-          X: expect.any(Float32Array),
-          Y: expect.any(Float32Array)
-        }),
-        varDataCache: expect.any(Object)
+        obsAnnotations: expect.any(Dataframe.Dataframe),
+        varAnnotations: expect.any(Dataframe.Dataframe),
+        obsLayout: expect.any(Dataframe.Dataframe),
+        varData: expect.any(Dataframe.Dataframe)
       })
     );
 
-    expect(universe.obsAnnotations).toHaveLength(nObs);
-    expect(_.keys(universe.obsNameToIndexMap)).toHaveLength(nObs);
-    expect(universe.obsLayout.X).toHaveLength(nObs);
-    expect(universe.obsLayout.Y).toHaveLength(nObs);
-    expect(universe.varAnnotations).toHaveLength(nVar);
-    expect(_.keys(universe.varNameToIndexMap)).toHaveLength(nVar);
+    expect(universe.obsAnnotations.dims).toEqual([
+      nObs,
+      REST.schema.schema.annotations.obs.length
+    ]);
+    expect(universe.obsLayout.dims).toEqual([nObs, 2]);
+    expect(universe.obsLayout.colIndex.keys()).toEqual(["X", "Y"]);
+    expect(universe.varAnnotations.dims).toEqual([
+      nVar,
+      REST.schema.schema.annotations.var.length
+    ]);
+    expect(universe.varData.isEmpty()).toBeTruthy();
   });
 });

client/__tests__/util/stateManager/world.test.js

@@ -1,13 +1,13 @@
 import _ from "lodash";
 import * as Universe from "../../../src/util/stateManager/universe";
 import * as World from "../../../src/util/stateManager/world";
+import * as Dataframe from "../../../src/util/dataframe";
 import Crossfilter from "../../../src/util/typedCrossfilter";
 import * as REST from "./sampleResponses";
 import {
   obsAnnoDimensionName,
   layoutDimensionName
 } from "../../../src/util/nameCreators";
-import * as kvCache from "../../../src/util/stateManager/keyvalcache";
 
 /*
 Helper - creates universe, world, corssfilter and dimensionMap from
@@ -16,7 +16,7 @@ the default REST test response.
 const defaultBigBang = () => {
   /* create unverse, world, crossfilter and dimensionMap */
   /* create universe */
-  const universe = Universe.createUniverseFromRestV02Response(
+  const universe = Universe.createUniverseFromResponse(
     REST.config,
     REST.schema,
     REST.annotationsObs,
@@ -40,7 +40,7 @@ const defaultBigBang = () => {
 
 describe("createWorldFromEntireUniverse", () => {
   test("create from REST sample", () => {
-    const universe = Universe.createUniverseFromRestV02Response(
+    const universe = Universe.createUniverseFromResponse(
       REST.config,
       REST.schema,
       REST.annotationsObs,
@@ -54,31 +54,13 @@ describe("createWorldFromEntireUniverse", () => {
 
     expect(world).toMatchObject(
       expect.objectContaining({
-        api: "0.2",
         nObs: universe.nObs,
         nVar: universe.nVar,
         schema: universe.schema,
         obsAnnotations: universe.obsAnnotations,
         varAnnotations: universe.varAnnotations,
         obsLayout: universe.obsLayout,
-
-        summary: expect.objectContaining({
-          obs: _(REST.schema.schema.annotations.obs)
-            .filter(v => v.name !== "name")
-            .keyBy("name")
-            .mapValues(() => expect.any(Object))
-            .value(),
-          var: _(REST.schema.schema.annotations.var)
-            .filter(v => v.name !== "name")
-            .keyBy("name")
-            .mapValues(() => expect.any(Object))
-            .value()
-        }),
-
-        varDataCache: expect.any(Object),
-
-        obsIndex: null, // null indicating full universe
-        obsBackIndex: null
+        varData: expect.any(Dataframe.Dataframe)
       })
     );
   });
@@ -111,51 +93,38 @@ describe("createWorldFromCurrentSelection", () => {
     */
 
     /* matchFilter must match the dimension filters above */
-    const matchFilter = val => val.field1 >= 0 && val.field1 < 5 && !val.field3;
-    const universeIndices = _()
-      .range(universe.nObs)
-      .filter(idx => matchFilter(universe.obsAnnotations[idx]))
-      .value();
-
-    const expected = {
-      nObs: universeIndices.length,
-      obsAnnotations: _.map(universeIndices, i => universe.obsAnnotations[i]),
-      obsLayout: {
-        X: new Float32Array(
-          _.map(universeIndices, i => universe.obsLayout.X[i])
-        ),
-        Y: new Float32Array(
-          _.map(universeIndices, i => universe.obsLayout.Y[i])
-        )
-      },
-      obsBackIndex: _.transform(
-        universeIndices,
-        (result, univIdx, worldIdx) => {
-          result[univIdx] = worldIdx;
-        },
-        new Uint32Array(universe.nObs).fill(-1)
-      ),
-      obsIndex: new Uint32Array(universeIndices)
+    const matchFilter = (df, row) => {
+      const field1 = df.at(row, "field1");
+      const field3 = df.at(row, "field3");
+      return field1 >= 0 && field1 < 5 && !field3;
     };
+    const matchingIndices = _()
+      .range(universe.nObs)
+      .filter(idx => matchFilter(universe.obsAnnotations, idx))
+      .value();
 
     expect(world).toMatchObject(
       expect.objectContaining({
-        api: "0.2",
-        nObs: expected.nObs,
+        nObs: matchingIndices.length,
         nVar: universe.nVar,
         schema: universe.schema,
-        obsAnnotations: expected.obsAnnotations,
+        obsAnnotations: expect.any(Dataframe.Dataframe),
         varAnnotations: universe.varAnnotations,
-        obsLayout: expected.obsLayout,
-        summary: {
-          obs: expect.any(Object) /* we could do better! */,
-          var: expect.any(Object) /* we could do better! */
-        },
-        varDataCache: expect.any(Object),
-        obsIndex: expected.obsIndex,
-        obsBackIndex: expected.obsBackIndex
+        obsLayout: expect.any(Dataframe.Dataframe),
+        varData: expect.any(Dataframe.Dataframe)
       })
     );
+
+    expect(world.obsAnnotations.rowIndex.keys()).toEqual(
+      new Int32Array(matchingIndices)
+    );
+    expect(world.obsAnnotations.colIndex.keys()).toEqual(
+      universe.obsAnnotations.colIndex.keys()
+    );
+    expect(world.obsLayout.rowIndex.keys()).toEqual(
+      new Int32Array(matchingIndices)
+    );
+    expect(world.obsLayout.colIndex.keys()).toEqual(["X", "Y"]);
   });
 });
 
@@ -193,58 +162,14 @@ describe("createObsDimensionMap", () => {
   });
 });
 
-describe("subsetVarData", () => {
-  test("when world eq universe", () => {
-    const { universe, world } = defaultBigBang();
-    /* create a mock varData array for subsetting */
-    const sourceVarData = new Float32Array(universe.nObs);
-
-    /* expect literally the same object back */
-    const result = World.subsetVarData(world, universe, sourceVarData);
-    expect(result).toBe(sourceVarData);
-  });
-
-  test("when world neq universe", () => {
-    const { universe, world, crossfilter, dimensionMap } = defaultBigBang();
-    /* create a mock varData array for subsetting */
-    const sourceVarData = Float32Array.from(_.range(universe.nObs));
-
-    /* mock a selection */
-    dimensionMap[obsAnnoDimensionName("field1")].filterRange([0, 5]);
-    dimensionMap[obsAnnoDimensionName("field3")].filterExact(false);
-
-    /* create the world from the selection */
-    const newWorld = World.createWorldFromCurrentSelection(
-      universe,
-      world,
-      crossfilter
-    );
-    expect(newWorld.obsIndex).toMatchObject(new Uint32Array([0, 2]));
-
-    /* expect a subset */
-    const result = World.subsetVarData(newWorld, universe, sourceVarData);
-    expect(result).not.toBe(sourceVarData);
-    expect(result).toHaveLength(newWorld.nObs);
-    /* check that we have expected source var content */
-    expect(result).toMatchObject(new Float32Array([0, 2]));
-  });
-});
-
-describe("createVarDimension", () => {
+describe("createVarDataDimension", () => {
   /* create default universe */
   const { world, crossfilter } = defaultBigBang();
-  /* create a mock var data cache */
-  const varDataCache = kvCache.set(
-    kvCache.create(),
+  world.varData = world.varData.withCol(
     "GENE",
     Float32Array.from(_.range(world.nObs))
   );
-  const result = World.createVarDimension(
-    world,
-    varDataCache,
-    crossfilter,
-    "GENE"
-  );
+  const result = World.createVarDataDimension(world, crossfilter, "GENE");
   expect(result).toBeInstanceOf(Crossfilter.ScalarDimension);
 });
 

client/__tests__/util/stateManager/worldUtil.test.js

@@ -2,16 +2,27 @@ import {
   countCategoryValues2D,
   clearCaches
 } from "../../../src/util/stateManager/worldUtil";
+import * as Dataframe from "../../../src/util/dataframe";
 
 describe("WorldUtil cache management", () => {
   test("empty", () => {
-    const count = countCategoryValues2D("a", "b", []);
+    const count = countCategoryValues2D(
+      "a",
+      "b",
+      new Dataframe.Dataframe([0, 0], [])
+    );
     expect(count).toMatchObject(new Map());
+    expect(count.size).toBe(0);
   });
 
   test("simple couts", () => {
-    const rows = [{ a: 0, b: false }, { a: 0, b: true }, { a: 1, b: false }];
-    const count = countCategoryValues2D("a", "b", rows);
+    const df = new Dataframe.Dataframe(
+      [3, 2],
+      [[0, 0, 1], [false, true, false]],
+      null,
+      new Dataframe.KeyIndex(["a", "b"])
+    );
+    const count = countCategoryValues2D("a", "b", df);
     expect(count).toMatchObject(
       new Map([
         [0, new Map([[true, 1], [false, 1]])],
@@ -22,16 +33,22 @@ describe("WorldUtil cache management", () => {
 
   test("memo cache clear", () => {
     clearCaches();
-    const row1 = [];
-    const row2 = [{ a: 0, b: false }, { a: 0, b: true }, { a: 1, b: false }];
-    const count1 = countCategoryValues2D("a", "b", row1);
-    const count2 = countCategoryValues2D("a", "b", row1);
-    const count3 = countCategoryValues2D("a", "b", []);
-    const count4 = countCategoryValues2D("a", "b", row2);
+    const df1 = new Dataframe.Dataframe([0, 0], []);
+    const df2 = new Dataframe.Dataframe(
+      [3, 2],
+      [[0, 0, 1], [false, true, false]],
+      null,
+      new Dataframe.KeyIndex(["a", "b"])
+    );
+
+    const count1 = countCategoryValues2D("a", "b", df1);
+    const count2 = countCategoryValues2D("a", "b", df1);
+    const count3 = countCategoryValues2D("a", "b", df1.clone());
+    const count4 = countCategoryValues2D("a", "b", df2);
 
     clearCaches();
-    const count10 = countCategoryValues2D("a", "b", row1);
-    const count11 = countCategoryValues2D("a", "b", row2);
+    const count10 = countCategoryValues2D("a", "b", df1);
+    const count11 = countCategoryValues2D("a", "b", df2);
 
     expect(count1).toEqual(count2);
     expect(count1).toEqual(count3);

client/__tests__/util/typedCrossfilter/bitArray.test.js

@@ -118,16 +118,16 @@ describe("selectionCount", () => {
     const dim2 = ba.allocDimension();
     expect(dim2).toBeDefined();
 
-    expect(ba.selectionCount).toEqual(0);
+    expect(ba.selectionCount()).toEqual(0);
     ba.selectAll(dim1);
-    expect(ba.selectionCount).toEqual(0);
+    expect(ba.selectionCount()).toEqual(0);
     ba.selectAll(dim2);
-    expect(ba.selectionCount).toEqual(defaultTestLength);
+    expect(ba.selectionCount()).toEqual(defaultTestLength);
 
     for (let i = 0; i < defaultTestLength; i += 1) {
       ba.deselectOne(dim1, i);
-      expect(ba.selectionCount).toEqual(defaultTestLength - i - 1);
-      expect(ba.selectionCount).toEqual(ba.countAllOnes());
+      expect(ba.selectionCount()).toEqual(defaultTestLength - i - 1);
+      expect(ba.selectionCount()).toEqual(ba.countAllOnes());
     }
 
     ba.freeDimension(dim1);

client/__tests__/util/typedCrossfilter/typedCrossfilter.test.js

@@ -119,14 +119,22 @@ function groupReduce(data, valueMap, valueReduce, valueInit) {
 }
 
 function groupCount(data, map) {
-  return groupReduce(data, map, (p, v) => p + 1, () => 0);
+  return groupReduce(data, map, p => p + 1, () => 0);
 }
 
 function groupSum(data, map) {
-  return groupReduce(data, map, (p, v) => (p += map(v)), () => 0);
+  return groupReduce(
+    data,
+    map,
+    (p, v) => {
+      p += map(v);
+      return p;
+    },
+    () => 0
+  );
 }
 
-var payments = null;
+let payments = null;
 beforeEach(() => {
   payments = crossfilter(someData);
 });
@@ -139,7 +147,7 @@ describe("typedCrossfilter", () => {
 
     const quantity = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.quantity,
+      (i, data) => data[i].quantity,
       Int32Array
     );
     expect(quantity).toBeDefined();
@@ -154,20 +162,23 @@ describe("typedCrossfilter", () => {
     expect(payments).toBeDefined();
     const quantity = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.quantity,
+      (i, data) => data[i].quantity,
       Int32Array
     );
     const tip = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.tip,
+      (i, data) => data[i].tip,
       Float32Array
     );
     const total = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.total,
+      (i, data) => data[i].total,
       Float32Array
     );
-    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
+    const type = payments.dimension(
+      crossfilter.EnumDimension,
+      (i, data) => data[i].type
+    );
 
     expect(quantity).toBeDefined();
     expect(tip).toBeDefined();
@@ -214,20 +225,18 @@ describe("typedCrossfilter", () => {
     expect(payments).toBeDefined();
     const quantity = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.quantity,
+      (i, data) => data[i].quantity,
       Int32Array
     );
     const tip = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.tip,
+      (i, data) => data[i].tip,
       Float32Array
     );
-    const total = payments.dimension(
-      crossfilter.ScalarDimension,
-      r => r.total,
-      Float32Array
+    const type = payments.dimension(
+      crossfilter.EnumDimension,
+      (i, data) => data[i].type
     );
-    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
 
     quantity.filterExact(1);
     expect(payments.countFiltered()).toEqual(
@@ -250,20 +259,23 @@ describe("typedCrossfilter", () => {
     expect(payments).toBeDefined();
     const quantity = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.quantity,
+      (i, data) => data[i].quantity,
       Int32Array
     );
     const tip = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.tip,
+      (i, data) => data[i].tip,
       Float32Array
     );
     const total = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.total,
+      (i, data) => data[i].total,
       Float32Array
     );
-    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
+    const type = payments.dimension(
+      crossfilter.EnumDimension,
+      (i, data) => data[i].type
+    );
 
     tip.filterRange([0, 91]);
     expect(payments.allFiltered()).toEqual(
@@ -291,20 +303,23 @@ describe("typedCrossfilter", () => {
     expect(payments).toBeDefined();
     const quantity = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.quantity,
+      (i, data) => data[i].quantity,
       Int32Array
     );
     const tip = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.tip,
+      (i, data) => data[i].tip,
       Float32Array
     );
     const total = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.total,
+      (i, data) => data[i].total,
       Float32Array
     );
-    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
+    const type = payments.dimension(
+      crossfilter.EnumDimension,
+      (i, data) => data[i].type
+    );
 
     type.filterEnum(["tab", "cash"]);
     expect(payments.allFiltered()).toEqual(
@@ -326,27 +341,30 @@ describe("typedCrossfilter", () => {
     expect(payments).toBeDefined();
     const quantity = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.quantity,
+      (i, data) => data[i].quantity,
       Int32Array
     );
     const tip = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.tip,
+      (i, data) => data[i].tip,
       Float32Array
     );
     const total = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.total,
+      (i, data) => data[i].total,
       Float32Array
     );
-    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
+    const type = payments.dimension(
+      crossfilter.EnumDimension,
+      (i, data) => data[i].type
+    );
 
     // Create a bunch of fake dimensions to ensure we can handle > 32
     let dimMap = {};
     for (let i = 0; i < 65; i++) {
       dimMap[i] = payments.dimension(
         crossfilter.ScalarDimension,
-        r => Math.random(),
+        () => Math.random(),
         Float32Array
       );
       expect(dimMap[i]).toBeDefined();
@@ -372,18 +390,21 @@ describe("typedCrossfilter", () => {
 
     const quantity = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.quantity,
+      (i, data) => data[i].quantity,
       Int32Array
     );
     const tip = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.tip,
+      (i, data) => data[i].tip,
       Int32Array
     );
-    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
+    const type = payments.dimension(
+      crossfilter.EnumDimension,
+      (i, data) => data[i].type
+    );
     const total = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.total,
+      (i, data) => data[i].total,
       Int32Array
     );
 
@@ -411,15 +432,18 @@ describe("typedCrossfilter", () => {
 
     const tip = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.tip,
+      (i, data) => data[i].tip,
       Int32Array
     );
     const totalX10 = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.total * 10,
+      (i, data) => data[i].total * 10,
       Int32Array
     );
-    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
+    const type = payments.dimension(
+      crossfilter.EnumDimension,
+      (i, data) => data[i].type
+    );
 
     const paymentsByTip_A = tip.group();
     const paymentsByTip_B = tip.group(r => 10 * r);
@@ -458,10 +482,13 @@ describe("typedCrossfilter", () => {
 
     const total = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.total,
+      (i, data) => data[i].total,
       Float32Array
     );
-    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
+    const type = payments.dimension(
+      crossfilter.EnumDimension,
+      (i, data) => data[i].type
+    );
 
     const paymentsByTotal = total.group();
     const paymentsByType = type.group();
@@ -499,15 +526,18 @@ describe("typedCrossfilter", () => {
 
     const tip = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.tip,
+      (i, data) => data[i].tip,
       Int32Array
     );
     const total = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.total,
+      (i, data) => data[i].total,
       Int32Array
     );
-    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
+    const type = payments.dimension(
+      crossfilter.EnumDimension,
+      (i, data) => data[i].type
+    );
 
     const paymentsByTip = tip.group();
     const paymentsByTotal = total.group();

client/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "cellxgene",
-  "version": "0.6.1",
+  "version": "0.7.0",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -1296,6 +1296,15 @@
       "integrity": "sha512-OtUw6JUTgxA2QoqqmrmQ7F2NYqiBPi/L2jqHyFtllhOUvXYQXf0Z1CYUinIfyT4bTCGmrA7gX9FvHA81uzCoVw==",
       "dev": true
     },
+    "agent-base": {
+      "version": "4.2.1",
+      "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.2.1.tgz",
+      "integrity": "sha512-JVwXMr9nHYTUXsBFKUqhJwvlcYU/blreOEUkhNR2eXZIvwd+c+o5V4MgDPKWnMS/56awN3TRzIP+KoPn+roQtg==",
+      "dev": true,
+      "requires": {
+        "es6-promisify": "^5.0.0"
+      }
+    },
     "ajv": {
       "version": "6.5.4",
       "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.5.4.tgz",
@@ -2807,6 +2816,12 @@
       "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==",
       "dev": true
     },
+    "check-more-types": {
+      "version": "2.24.0",
+      "resolved": "https://registry.npmjs.org/check-more-types/-/check-more-types-2.24.0.tgz",
+      "integrity": "sha1-FCD/sQ/URNz8ebQ4kbv//TKoRgA=",
+      "dev": true
+    },
     "chokidar": {
       "version": "2.0.4",
       "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-2.0.4.tgz",
@@ -4118,6 +4133,12 @@
         "is-obj": "^1.0.0"
       }
     },
+    "duplexer": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.1.tgz",
+      "integrity": "sha1-rOb/gIwc5mtX0ev5eXessCM0z8E=",
+      "dev": true
+    },
     "duplexer3": {
       "version": "0.1.4",
       "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz",
@@ -4278,6 +4299,15 @@
       "integrity": "sha512-n6wvpdE43VFtJq+lUDYDBFUwV8TZbuGXLV4D6wKafg13ldznKsyEvatubnmUe31zcvelSzOHF+XbaT+Bl9ObDg==",
       "dev": true
     },
+    "es6-promisify": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz",
+      "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=",
+      "dev": true,
+      "requires": {
+        "es6-promise": "^4.0.3"
+      }
+    },
     "escape-html": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
@@ -4767,6 +4797,21 @@
       "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=",
       "dev": true
     },
+    "event-stream": {
+      "version": "3.3.4",
+      "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-3.3.4.tgz",
+      "integrity": "sha1-SrTJoPWlTbkzi0w02Gv86PSzVXE=",
+      "dev": true,
+      "requires": {
+        "duplexer": "~0.1.1",
+        "from": "~0",
+        "map-stream": "~0.1.0",
+        "pause-stream": "0.0.11",
+        "split": "0.3",
+        "stream-combiner": "~0.0.4",
+        "through": "~2.3.1"
+      }
+    },
     "events": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz",
@@ -4963,6 +5008,35 @@
         "is-extglob": "^1.0.0"
       }
     },
+    "extract-zip": {
+      "version": "1.6.7",
+      "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-1.6.7.tgz",
+      "integrity": "sha1-qEC0uK9kAyZMjbV/Txp0Mz74H+k=",
+      "dev": true,
+      "requires": {
+        "concat-stream": "1.6.2",
+        "debug": "2.6.9",
+        "mkdirp": "0.5.1",
+        "yauzl": "2.4.1"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "2.6.9",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+          "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+          "dev": true,
+          "requires": {
+            "ms": "2.0.0"
+          }
+        },
+        "ms": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+          "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=",
+          "dev": true
+        }
+      }
+    },
     "extsprintf": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
@@ -5023,6 +5097,15 @@
         }
       }
     },
+    "fd-slicer": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.0.1.tgz",
+      "integrity": "sha1-i1vL2ewyfFBBv5qwI/1nUPEXfmU=",
+      "dev": true,
+      "requires": {
+        "pend": "~1.2.0"
+      }
+    },
     "figures": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz",
@@ -5229,6 +5312,12 @@
       "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=",
       "dev": true
     },
+    "from": {
+      "version": "0.1.7",
+      "resolved": "https://registry.npmjs.org/from/-/from-0.1.7.tgz",
+      "integrity": "sha1-g8YK/Fi5xWmXAH7Rp2izqzA6RP4=",
+      "dev": true
+    },
     "from2": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz",
@@ -5277,7 +5366,8 @@
         "ansi-regex": {
           "version": "2.1.1",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "aproba": {
           "version": "1.2.0",
@@ -5298,12 +5388,14 @@
         "balanced-match": {
           "version": "1.0.0",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "brace-expansion": {
           "version": "1.1.11",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "balanced-match": "^1.0.0",
             "concat-map": "0.0.1"
@@ -5318,17 +5410,20 @@
         "code-point-at": {
           "version": "1.1.0",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "concat-map": {
           "version": "0.0.1",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "console-control-strings": {
           "version": "1.1.0",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "core-util-is": {
           "version": "1.0.2",
@@ -5445,7 +5540,8 @@
         "inherits": {
           "version": "2.0.3",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "ini": {
           "version": "1.3.5",
@@ -5457,6 +5553,7 @@
           "version": "1.0.0",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "number-is-nan": "^1.0.0"
           }
@@ -5471,6 +5568,7 @@
           "version": "3.0.4",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "brace-expansion": "^1.1.7"
           }
@@ -5478,12 +5576,14 @@
         "minimist": {
           "version": "0.0.8",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "minipass": {
           "version": "2.2.4",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "safe-buffer": "^5.1.1",
             "yallist": "^3.0.0"
@@ -5502,6 +5602,7 @@
           "version": "0.5.1",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "minimist": "0.0.8"
           }
@@ -5582,7 +5683,8 @@
         "number-is-nan": {
           "version": "1.0.1",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "object-assign": {
           "version": "4.1.1",
@@ -5594,6 +5696,7 @@
           "version": "1.4.0",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "wrappy": "1"
           }
@@ -5679,7 +5782,8 @@
         "safe-buffer": {
           "version": "5.1.1",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "safer-buffer": {
           "version": "2.1.2",
@@ -5715,6 +5819,7 @@
           "version": "1.0.2",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "code-point-at": "^1.0.0",
             "is-fullwidth-code-point": "^1.0.0",
@@ -5734,6 +5839,7 @@
           "version": "3.0.1",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "ansi-regex": "^2.0.0"
           }
@@ -5777,12 +5883,14 @@
         "wrappy": {
           "version": "1.0.2",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "yallist": {
           "version": "3.0.2",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         }
       }
     },
@@ -6127,6 +6235,12 @@
         "minimalistic-crypto-utils": "^1.0.1"
       }
     },
+    "hoek": {
+      "version": "5.0.4",
+      "resolved": "https://registry.npmjs.org/hoek/-/hoek-5.0.4.tgz",
+      "integrity": "sha512-Alr4ZQgoMlnere5FZJsIyfIjORBqZll5POhDsF4q64dPuJR6rNxXdDxtHSQq8OXRurhmx+PWYEE8bXRROY8h0w==",
+      "dev": true
+    },
     "hoist-non-react-statics": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.1.0.tgz",
@@ -6295,6 +6409,16 @@
       "integrity": "sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM=",
       "dev": true
     },
+    "https-proxy-agent": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz",
+      "integrity": "sha512-HPCTS1LW51bcyMYbxUIOO4HEOlQ1/1qRaFWcyxvwaqUS9TY88aoEuHUY33kuAh1YhVVaDQhLZsnPd+XNARWZlQ==",
+      "dev": true,
+      "requires": {
+        "agent-base": "^4.1.0",
+        "debug": "^3.1.0"
+      }
+    },
     "iconv-lite": {
       "version": "0.4.24",
       "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
@@ -6825,6 +6949,15 @@
       "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
       "dev": true
     },
+    "isemail": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/isemail/-/isemail-3.2.0.tgz",
+      "integrity": "sha512-zKqkK+O+dGqevc93KNsbZ/TqTUFd46MwWjYOoMrjIMZ51eU7DtQG3Wmd9SQQT7i7RVnuTPEiYEWHU3MSbxC1Tg==",
+      "dev": true,
+      "requires": {
+        "punycode": "2.x.x"
+      }
+    },
     "isexe": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
@@ -9078,6 +9211,17 @@
         }
       }
     },
+    "joi": {
+      "version": "13.7.0",
+      "resolved": "https://registry.npmjs.org/joi/-/joi-13.7.0.tgz",
+      "integrity": "sha512-xuY5VkHfeOYK3Hdi91ulocfuFopwgbSORmIwzcwHKESQhC7w1kD5jaVSPnqDxS2I8t3RZ9omCKAxNwXN5zG1/Q==",
+      "dev": true,
+      "requires": {
+        "hoek": "5.x.x",
+        "isemail": "3.x.x",
+        "topo": "3.x.x"
+      }
+    },
     "js-levenshtein": {
       "version": "1.1.4",
       "resolved": "https://registry.npmjs.org/js-levenshtein/-/js-levenshtein-1.1.4.tgz",
@@ -9240,6 +9384,12 @@
         "package-json": "^4.0.0"
       }
     },
+    "lazy-ass": {
+      "version": "1.6.0",
+      "resolved": "https://registry.npmjs.org/lazy-ass/-/lazy-ass-1.6.0.tgz",
+      "integrity": "sha1-eZllXoZGwX8In90YfRUNMyTVRRM=",
+      "dev": true
+    },
     "lcid": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/lcid/-/lcid-2.0.0.tgz",
@@ -9521,6 +9671,12 @@
       "integrity": "sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0=",
       "dev": true
     },
+    "map-stream": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.1.0.tgz",
+      "integrity": "sha1-5WqpTEyAVaFkBKBnS3jyFffI4ZQ=",
+      "dev": true
+    },
     "map-visit": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/map-visit/-/map-visit-1.0.0.tgz",
@@ -10533,6 +10689,15 @@
         }
       }
     },
+    "pause-stream": {
+      "version": "0.0.11",
+      "resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz",
+      "integrity": "sha1-/lo0sMvOErWqaitAPuLnO2AvFEU=",
+      "dev": true,
+      "requires": {
+        "through": "~2.3"
+      }
+    },
     "pbkdf2": {
       "version": "3.0.17",
       "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.0.17.tgz",
@@ -10546,6 +10711,12 @@
         "sha.js": "^2.4.8"
       }
     },
+    "pend": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
+      "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=",
+      "dev": true
+    },
     "performance-now": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz",
@@ -10791,12 +10962,27 @@
         "ipaddr.js": "1.8.0"
       }
     },
+    "proxy-from-env": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.0.0.tgz",
+      "integrity": "sha1-M8UDmPcOp+uW0h97gXYwpVeRx+4=",
+      "dev": true
+    },
     "prr": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz",
       "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=",
       "dev": true
     },
+    "ps-tree": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/ps-tree/-/ps-tree-1.2.0.tgz",
+      "integrity": "sha512-0VnamPPYHl4uaU/nSFeZZpR21QAWRz+sRv4iW9+v/GS/J5U5iZB5BNN6J0RMoOvdx2gWM2+ZFMIm58q24e4UYA==",
+      "dev": true,
+      "requires": {
+        "event-stream": "=3.3.4"
+      }
+    },
     "pseudomap": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
@@ -10850,6 +11036,48 @@
       "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==",
       "dev": true
     },
+    "puppeteer": {
+      "version": "1.12.2",
+      "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-1.12.2.tgz",
+      "integrity": "sha512-xWSyCeD6EazGlfnQweMpM+Hs6X6PhUYhNTHKFj/axNZDq4OmrVERf70isBf7HsnFgB3zOC1+23/8+wCAZYg+Pg==",
+      "dev": true,
+      "requires": {
+        "debug": "^4.1.0",
+        "extract-zip": "^1.6.6",
+        "https-proxy-agent": "^2.2.1",
+        "mime": "^2.0.3",
+        "progress": "^2.0.1",
+        "proxy-from-env": "^1.0.0",
+        "rimraf": "^2.6.1",
+        "ws": "^6.1.0"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "4.1.1",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
+          "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
+          "dev": true,
+          "requires": {
+            "ms": "^2.1.1"
+          }
+        },
+        "mime": {
+          "version": "2.4.0",
+          "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.0.tgz",
+          "integrity": "sha512-ikBcWwyqXQSHKtciCcctu9YfPbFYZ4+gbHEmE0Q8jzcTYQg5dHCr3g2wwAZjPoJfQVXZq6KXAjpXOTf5/cjT7w==",
+          "dev": true
+        },
+        "ws": {
+          "version": "6.1.4",
+          "resolved": "https://registry.npmjs.org/ws/-/ws-6.1.4.tgz",
+          "integrity": "sha512-eqZfL+NE/YQc1/ZynhojeV8q+H050oR8AZ2uIev7RU10svA9ZnJUddHcOUZTJLinZ9yEfdA2kSATS2qZK5fhJA==",
+          "dev": true,
+          "requires": {
+            "async-limiter": "~1.0.0"
+          }
+        }
+      }
+    },
     "qs": {
       "version": "6.5.2",
       "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz",
@@ -11817,6 +12045,12 @@
       "resolved": "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz",
       "integrity": "sha1-P4Yt+pGrdmsUiF700BEkv9oHT7Q="
     },
+    "rx": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/rx/-/rx-4.1.0.tgz",
+      "integrity": "sha1-pfE/957zt0D+MKqAP7CfmIBdR4I=",
+      "dev": true
+    },
     "rxjs": {
       "version": "6.4.0",
       "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.4.0.tgz",
@@ -12637,6 +12871,15 @@
       "integrity": "sha512-TfOfPcYGBB5sDuPn3deByxPhmfegAhpDYKSOXZQN81Oyrrif8ZCodOLzK3AesELnCx03kikhyDwh0pfvvQvF8w==",
       "dev": true
     },
+    "split": {
+      "version": "0.3.3",
+      "resolved": "https://registry.npmjs.org/split/-/split-0.3.3.tgz",
+      "integrity": "sha1-zQ7qXmOiEd//frDwkcQTPi0N0o8=",
+      "dev": true,
+      "requires": {
+        "through": "2"
+      }
+    },
     "split-string": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz",
@@ -12684,6 +12927,63 @@
       "integrity": "sha512-MTX+MeG5U994cazkjd/9KNAapsHnibjMLnfXodlkXw76JEea0UiNzrqidzo1emMwk7w5Qhc9jd4Bn9TBb1MFwA==",
       "dev": true
     },
+    "start-server-and-test": {
+      "version": "1.7.11",
+      "resolved": "https://registry.npmjs.org/start-server-and-test/-/start-server-and-test-1.7.11.tgz",
+      "integrity": "sha512-651SCOfhPT65Xjhecvx/ZMJs8UOd5VItjjmpYH95aM6Hr4P8N8UIcxEgDhY/aaVmthACH7qFTqs4EA/KHykjtw==",
+      "dev": true,
+      "requires": {
+        "bluebird": "3.5.3",
+        "check-more-types": "2.24.0",
+        "debug": "3.2.6",
+        "execa": "0.11.0",
+        "lazy-ass": "1.6.0",
+        "ps-tree": "1.2.0",
+        "wait-on": "3.2.0"
+      },
+      "dependencies": {
+        "bluebird": {
+          "version": "3.5.3",
+          "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.3.tgz",
+          "integrity": "sha512-/qKPUQlaW1OyR51WeCPBvRnAlnZFUJkCSG5HzGnuIqhgyJtF+T94lFnn33eiazjRm2LAHVy2guNnaq48X9SJuw==",
+          "dev": true
+        },
+        "execa": {
+          "version": "0.11.0",
+          "resolved": "https://registry.npmjs.org/execa/-/execa-0.11.0.tgz",
+          "integrity": "sha512-k5AR22vCt1DcfeiRixW46U5tMLtBg44ssdJM9PiXw3D8Bn5qyxFCSnKY/eR22y+ctFDGPqafpaXg2G4Emyua4A==",
+          "dev": true,
+          "requires": {
+            "cross-spawn": "^6.0.0",
+            "get-stream": "^4.0.0",
+            "is-stream": "^1.1.0",
+            "npm-run-path": "^2.0.0",
+            "p-finally": "^1.0.0",
+            "signal-exit": "^3.0.0",
+            "strip-eof": "^1.0.0"
+          }
+        },
+        "get-stream": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz",
+          "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==",
+          "dev": true,
+          "requires": {
+            "pump": "^3.0.0"
+          }
+        },
+        "pump": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
+          "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
+          "dev": true,
+          "requires": {
+            "end-of-stream": "^1.1.0",
+            "once": "^1.3.1"
+          }
+        }
+      }
+    },
     "static-extend": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/static-extend/-/static-extend-0.1.2.tgz",
@@ -12727,6 +13027,15 @@
         "readable-stream": "^2.0.2"
       }
     },
+    "stream-combiner": {
+      "version": "0.0.4",
+      "resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.0.4.tgz",
+      "integrity": "sha1-TV5DPBhSYd3mI8o/RMWGvPXErRQ=",
+      "dev": true,
+      "requires": {
+        "duplexer": "~0.1.1"
+      }
+    },
     "stream-each": {
       "version": "1.2.3",
       "resolved": "https://registry.npmjs.org/stream-each/-/stream-each-1.2.3.tgz",
@@ -13174,6 +13483,23 @@
         }
       }
     },
+    "topo": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/topo/-/topo-3.0.3.tgz",
+      "integrity": "sha512-IgpPtvD4kjrJ7CRA3ov2FhWQADwv+Tdqbsf1ZnPUSAtCJ9e1Z44MmoSGDXGk4IppoZA7jd/QRkNddlLJWlUZsQ==",
+      "dev": true,
+      "requires": {
+        "hoek": "6.x.x"
+      },
+      "dependencies": {
+        "hoek": {
+          "version": "6.1.2",
+          "resolved": "https://registry.npmjs.org/hoek/-/hoek-6.1.2.tgz",
+          "integrity": "sha512-6qhh/wahGYZHFSFw12tBbJw5fsAhhwrrG/y3Cs0YMTv2WzMnL0oLPnQJjv1QJvEfylRSOFuP+xCu+tdx0tD16Q==",
+          "dev": true
+        }
+      }
+    },
     "toposort": {
       "version": "1.0.7",
       "resolved": "https://registry.npmjs.org/toposort/-/toposort-1.0.7.tgz",
@@ -13695,6 +14021,27 @@
         "browser-process-hrtime": "^0.1.2"
       }
     },
+    "wait-on": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/wait-on/-/wait-on-3.2.0.tgz",
+      "integrity": "sha512-QUGNKlKLDyY6W/qHdxaRlXUAgLPe+3mLL/tRByHpRNcHs/c7dZXbu+OnJWGNux6tU1WFh/Z8aEwvbuzSAu79Zg==",
+      "dev": true,
+      "requires": {
+        "core-js": "^2.5.7",
+        "joi": "^13.0.0",
+        "minimist": "^1.2.0",
+        "request": "^2.88.0",
+        "rx": "^4.1.0"
+      },
+      "dependencies": {
+        "minimist": {
+          "version": "1.2.0",
+          "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
+          "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=",
+          "dev": true
+        }
+      }
+    },
     "walker": {
       "version": "1.0.7",
       "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.7.tgz",
@@ -14575,6 +14922,15 @@
           "dev": true
         }
       }
+    },
+    "yauzl": {
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.4.1.tgz",
+      "integrity": "sha1-lSj0QtqxsihOWLQ3m7GU4i4MQAU=",
+      "dev": true,
+      "requires": {
+        "fd-slicer": "~1.0.1"
+      }
     }
   }
 }

client/package.json

@@ -1,17 +1,20 @@
 {
   "name": "cellxgene",
-  "version": "0.6.1",
+  "version": "0.7.0",
   "license": "MIT",
   "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
   "repository": "https://github.com/chanzuckerberg/cellxgene",
   "scripts": {
     "backend-dev": "python3.6 -m venv cellxgene && source cellxgene/bin/activate && yes | pip uninstall cellxgene || true && pip install -e .. && cellxgene launch ",
     "build": "npm run clean && webpack --config configuration/webpack/webpack.config.prod.js",
-    "dev": "npm run clean && webpack --config configuration/webpack/webpack.config.dev.js",
     "clean": "rimraf build",
-    "start": "node server/development.js",
+    "dev": "npm run clean && webpack --config configuration/webpack/webpack.config.dev.js",
+    "e2e": "jest e2e",
     "lint": "eslint src",
-    "test": "jest"
+    "smoke-test": "start-server-and-test start-server-for-test :5000 e2e",
+    "start": "node server/development.js",
+    "start-server-for-test": "cellxgene launch -p 5000 ../example-dataset/pbmc3k.h5ad",
+    "unit-test": "jest --testPathIgnorePatterns e2e"
   },
   "engineStrict": true,
   "engines": {
@@ -97,8 +100,10 @@
     "jest": "^24.1.0",
     "json-loader": "^0.5.4",
     "mini-css-extract-plugin": "^0.4.1",
+    "puppeteer": "^1.12.1",
     "rimraf": "^2.6.3",
     "serve-favicon": "^2.3.0",
+    "start-server-and-test": "^1.7.11",
     "style-loader": "^0.23.1",
     "sw-precache-webpack-plugin": "^0.11.5",
     "url-loader": "^1.1.0",

client/src/actions/index.js

@@ -1,12 +1,11 @@
 // jshint esversion: 6
 import _ from "lodash";
 import * as globals from "../globals";
-import { Universe, kvCache } from "../util/stateManager";
+import { Universe } from "../util/stateManager";
 import {
   catchErrorsWrap,
   doJsonRequest,
   doBinaryRequest,
-  rangeEncodeIndices,
   dispatchNetworkErrorMessageToUser
 } from "../util/actionHelpers";
 
@@ -40,7 +39,7 @@ const doInitialDataLoad = () =>
       /* set config defaults */
       const config = { ...globals.configDefaults, ...results[0].config };
       const [, schema, obsAnno, varAnno, obsLayout] = [...results];
-      const universe = Universe.createUniverseFromRestV02Response(
+      const universe = Universe.createUniverseFromResponse(
         config,
         schema,
         obsAnno,
@@ -130,9 +129,9 @@ async function _doRequestExpressionData(dispatch, getState, genes) {
   let expressionData = _.transform(
     genes,
     (expData, g) => {
-      const data = kvCache.get(universe.varDataCache, g);
+      const data = universe.varData.col(g);
       if (data) {
-        expData[g] = data;
+        expData[g] = data.asArray();
       }
     },
     {}
@@ -170,7 +169,7 @@ function requestSingleGeneExpressionCountsForColoringPOST(gene) {
         type: "color by expression",
         gene,
         data: {
-          [gene]: kvCache.get(world.varDataCache, gene)
+          [gene]: world.varData.col(gene).asArray()
         }
       });
     } catch (error) {
@@ -193,7 +192,7 @@ const requestUserDefinedGene = gene => async (dispatch, getState) => {
       type: "request user defined gene success",
       data: {
         genes: [gene],
-        expression: kvCache.get(world.varDataCache, gene)
+        expression: world.varData.col(gene).asArray()
       }
     });
   } catch (error) {
@@ -242,12 +241,18 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
     */
     const state = getState();
     const { universe } = state.controls;
-    const set1ByIndex = rangeEncodeIndices(
-      _.map(set1, s => universe.obsNameToIndexMap[s])
-    );
-    const set2ByIndex = rangeEncodeIndices(
-      _.map(set2, s => universe.obsNameToIndexMap[s])
-    );
+
+    // Legal values are null, Array or TypedArray.  Null is initial state.
+    if (!set1) set1 = [];
+    if (!set2) set2 = [];
+
+    // These lines ensure that we convert any TypedArray to an Array.
+    // This is necessary because JSON.stringify() does some very strange
+    // things with TypedArrays (they are marshalled to JSON objects, rather
+    // than being marshalled as a JSON array).
+    set1 = Array.isArray(set1) ? set1 : Array.from(set1);
+    set2 = Array.isArray(set2) ? set2 : Array.from(set2);
+
     const res = await fetch(
       `${globals.API.prefix}${globals.API.version}diffexp/obs`,
       {
@@ -259,8 +264,8 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
         body: JSON.stringify({
           mode: "topN",
           count: num_genes,
-          set1: { filter: { obs: { index: set1ByIndex } } },
-          set2: { filter: { obs: { index: set2ByIndex } } }
+          set1: { filter: { obs: { index: set1 } } },
+          set2: { filter: { obs: { index: set2 } } }
         })
       }
     );
@@ -271,7 +276,9 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
 
     const data = await res.json();
     // result is [ [varIdx, ...], ... ]
-    const topNGenes = _.map(data, r => universe.varAnnotations[r[0]].name);
+    const topNGenes = _.map(data, r =>
+      universe.varAnnotations.at(r[0], "name")
+    );
 
     /*
     Kick off secondary action to fetch all of the expression data for the

client/src/components/brushableHistogram/index.js

@@ -10,7 +10,6 @@ import { Button, ButtonGroup, Tooltip } from "@blueprintjs/core";
 import { connect } from "react-redux";
 import * as d3 from "d3";
 import memoize from "memoize-one";
-import { kvCache } from "../../util/stateManager";
 import * as globals from "../../globals";
 import actions from "../../actions";
 import finiteExtent from "../../util/finiteExtent";
@@ -21,7 +20,6 @@ import finiteExtent from "../../util/finiteExtent";
   scatterplotYYaccessor: state.controls.scatterplotYYaccessor,
   crossfilter: state.controls.crossfilter,
   differential: state.differential,
-  initializeRanges: _.get(state.controls.world, "summary.obs"),
   colorAccessor: state.controls.colorAccessor,
   colorScale: state.controls.colorScale,
   obsAnnotations: _.get(state.controls.world, "obsAnnotations", null)
@@ -35,9 +33,11 @@ class HistogramBrush extends React.Component {
       .scaleLinear()
       .range([this.height - this.marginBottom, 0]);
 
-    if (obsAnnotations[0][field] !== undefined) {
+    if (obsAnnotations.hasCol(field)) {
       // recalculate expensive stuff
-      const allValuesForContinuousFieldAsArray = _.map(obsAnnotations, field);
+      const allValuesForContinuousFieldAsArray = obsAnnotations
+        .col(field)
+        .asArray();
 
       histogramCache.x = d3
         .scaleLinear()
@@ -50,9 +50,8 @@ class HistogramBrush extends React.Component {
         .thresholds(40)(allValuesForContinuousFieldAsArray);
 
       histogramCache.numValues = allValuesForContinuousFieldAsArray.length;
-    } else if (kvCache.get(world.varDataCache, field)) {
-      /* it's not in observations, so it's a gene, but let's check to make sure */
-      const varValues = kvCache.get(world.varDataCache, field);
+    } else if (world.varData.hasCol(field)) {
+      const varValues = world.varData.col(field).asArray();
 
       histogramCache.x = d3
         .scaleLinear()
@@ -141,21 +140,15 @@ class HistogramBrush extends React.Component {
   }
 
   handleColorAction() {
-    const {
-      obsAnnotations,
-      dispatch,
-      field,
-      world,
-      initializeRanges
-    } = this.props;
+    const { obsAnnotations, dispatch, field, world, ranges } = this.props;
 
-    if (obsAnnotations[0][field]) {
+    if (obsAnnotations.hasCol(field)) {
       dispatch({
         type: "color by continuous metadata",
         colorAccessor: field,
-        rangeMaxForColorAccessor: initializeRanges[field].range.max
+        rangeForColorAccessor: ranges
       });
-    } else if (kvCache.get(world.varDataCache, field)) {
+    } else if (world.varData.hasCol(field)) {
       dispatch(actions.requestSingleGeneExpressionCountsForColoringPOST(field));
     }
   }
@@ -235,6 +228,7 @@ class HistogramBrush extends React.Component {
     d3.select(svgRef)
       .append("g")
       .attr("class", "brush")
+      .attr("data-testid", `${svgRef.id}-brush`)
       .call(
         d3
           .brushX()
@@ -279,6 +273,8 @@ class HistogramBrush extends React.Component {
     return (
       <div
         id={`histogram_${field}`}
+        data-testid={`histogram-${field}`}
+        data-testclass={isDiffExp ? `histogram-diffexp` : ""}
         style={{
           padding: globals.leftSidebarSectionPadding,
           backgroundColor: zebra ? globals.lightestGrey : "white"

client/src/components/categorical/value.js

@@ -65,7 +65,11 @@ class CategoryValue extends React.Component {
       })[0].categories;
     }
 
-    if (colorAccessor && !isColorBy) {
+    if (
+      colorAccessor &&
+      !isColorBy &&
+      categoricalSelectionState[colorAccessor]
+    ) {
       occupancy = countCategoryValues2D(
         metadataField,
         colorAccessor,

client/src/components/continuous/continuous.js

@@ -9,8 +9,7 @@ import * as globals from "../../globals";
 import HistogramBrush from "../brushableHistogram";
 
 @connect(state => ({
-  ranges: _.get(state.controls.world, "summary.obs", null),
-  metadata: _.get(state.controls.world, "obsAnnotations", null),
+  obsAnnotations: _.get(state.controls.world, "obsAnnotations", null),
   colorAccessor: state.controls.colorAccessor,
   colorScale: state.controls.colorScale,
   selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null),
@@ -29,17 +28,18 @@ class Continuous extends React.Component {
 
   handleColorAction(key) {
     return () => {
-      const { dispatch, ranges } = this.props;
+      const { dispatch, obsAnnotations } = this.props;
+      const summary = obsAnnotations.col(key).summarize();
       dispatch({
         type: "color by continuous metadata",
         colorAccessor: key,
-        rangeMaxForColorAccessor: ranges[key].range.max
+        rangeForColorAccessor: summary
       });
     };
   }
 
   render() {
-    const { ranges, obsAnnotations, schema } = this.props;
+    const { obsAnnotations, schema } = this.props;
     if (schema && !this.continuousChecked) {
       this.hasContinuous = _.some(
         schema.annotations.obs,
@@ -63,24 +63,34 @@ class Continuous extends React.Component {
             Continuous metadata
           </p>
         ) : null}
-        {_.map(ranges, (value, key) => {
-          const isColorField = key.includes("color") || key.includes("Color");
-          zebra += 1;
-          if (value.range && key !== "name" && !isColorField) {
-            return (
-              <HistogramBrush
-                key={key}
-                field={key}
-                isObs
-                zebra={zebra % 2 === 0}
-                fieldValues={obsAnnotations}
-                ranges={value.range}
-                handleColorAction={this.handleColorAction(key).bind(this)}
-              />
-            );
-          }
-          return null;
-        })}
+        {obsAnnotations
+          ? _.map(obsAnnotations.colIndex.keys(), key => {
+              const summary = obsAnnotations.col(key).summarize();
+              const isColorField =
+                key.includes("color") || key.includes("Color");
+              const nonFiniteExtent =
+                summary.min === undefined || summary.max === undefined;
+              zebra += 1;
+              if (
+                !summary.categorical &&
+                key !== "name" &&
+                !isColorField &&
+                !nonFiniteExtent
+              ) {
+                return (
+                  <HistogramBrush
+                    key={key}
+                    field={key}
+                    isObs
+                    zebra={zebra % 2 === 0}
+                    ranges={summary}
+                    handleColorAction={this.handleColorAction(key).bind(this)}
+                  />
+                );
+              }
+              return null;
+            })
+          : null}
       </div>
     );
   }

client/src/components/geneExpression/cellSetButtons.js

@@ -1,8 +1,8 @@
 // jshint esversion: 6
 import React from "react";
-import _ from "lodash";
 import { AnchorButton, Tooltip } from "@blueprintjs/core";
 import { connect } from "react-redux";
+import { World } from "../../util/stateManager";
 
 @connect()
 class CellSetButton extends React.Component {
@@ -14,7 +14,11 @@ class CellSetButton extends React.Component {
       eitherCellSetOneOrTwo
     } = this.props;
 
-    const set = _.map(crossfilter.allFiltered(), "name");
+    // Reducer and components assume that value will be null if
+    // no selection made.  World..getSelectedByIndex() returns a
+    // zero length TypedArray when nothing is selected.
+    let set = World.getSelectedByIndex(crossfilter);
+    if (set.length === 0) set = null;
 
     if (!differential.diffExp) {
       /* diffexp needs to be cleared before we store a new set */
@@ -38,6 +42,7 @@ class CellSetButton extends React.Component {
           type="button"
           disabled={differential.diffExp}
           onClick={this.set.bind(this)}
+          data-testid={`cellset-button-${eitherCellSetOneOrTwo}`}
         >
           {eitherCellSetOneOrTwo}
           {": "}

client/src/components/geneExpression/expressionButtons.js

@@ -68,6 +68,7 @@ class Expression extends React.Component {
               style={{ marginTop: 10 }}
               disabled={!haveBothCellSets}
               intent="primary"
+              data-testid="diffexp-button"
               loading={differential.loading}
               fill
               type="button"

client/src/components/geneExpression/index.js

@@ -29,8 +29,7 @@ const renderGene = (fuzzySortResult, { handleClick, modifiers, query }) => {
     return null;
   }
   /* the fuzzysort wraps the object with other properties, like a score */
-  const gene = fuzzySortResult.obj;
-  const text = gene.name;
+  const geneName = fuzzySortResult.target;
 
   return (
     <MenuItem
@@ -39,39 +38,30 @@ const renderGene = (fuzzySortResult, { handleClick, modifiers, query }) => {
       // Use of annotations in this way is incorrect and dataset specific.
       // See https://github.com/chanzuckerberg/cellxgene/issues/483
       // label={gene.n_counts}
-      key={gene.name}
-      onClick={g => {
+      key={geneName}
+      onClick={g =>
         /* this fires when user clicks a menu item */
-        handleClick(g);
-      }}
-      text={text}
+        handleClick(g)
+      }
+      text={geneName}
     />
   );
 };
 
-const filterGenes = (query, genes) => {
+const filterGenes = (query, genes) =>
   /* fires on load, once, and then for each character typed into the input */
-  return fuzzysort.go(query, genes, {
-    key: "name",
+  fuzzysort.go(query, genes, {
     limit: 5,
     threshold: -10000 // don't return bad results
   });
-};
 
 @connect(state => {
-  const metadata = _.get(state.controls.world, "obsAnnotations", null);
-  const ranges = _.get(state.controls.world, "summary.obs", null);
-  const initializeRanges = _.get(state.controls.world, "summary.obs");
-
   return {
-    ranges,
-    metadata,
-    initializeRanges,
+    obsAnnotations: _.get(state.controls.world, "obsAnnotations", null),
     userDefinedGenes: state.controls.userDefinedGenes,
     userDefinedGenesLoading: state.controls.userDefinedGenesLoading,
     world: state.controls.world,
     colorAccessor: state.controls.colorAccessor,
-    allGeneNames: state.controls.allGeneNames,
     differential: state.differential
   };
 })
@@ -84,6 +74,37 @@ class GeneExpression extends React.Component {
     };
   }
 
+  placeholderGeneNames() {
+    /*
+    return a string containing gene name suggestions for use as a user hint.
+    Eg.,    Apod, Cd74, ...
+    Will return a max of 3 genes, totalling 15 characters in length.
+    Randomly selects gene names.
+
+    NOTE: the random selection means it will re-render constantly.
+    */
+    const { world } = this.props;
+    const { varAnnotations } = world;
+    const geneNames = varAnnotations.col("name").asArray();
+    if (geneNames.length > 0) {
+      const placeholder = [];
+      let len = geneNames.length;
+      const maxGeneNameCount = 3;
+      const maxStrLength = 15;
+      len = len < maxGeneNameCount ? len : maxGeneNameCount;
+      for (let i = 0, strLen = 0; i < len && strLen < maxStrLength; i += 1) {
+        const deal = Math.floor(Math.random() * geneNames.length);
+        const geneName = geneNames[deal];
+        placeholder.push(geneName);
+        strLen += geneName.length + 2; // '2' is the length of a comma and space
+      }
+      placeholder.push("...");
+      return placeholder.join(", ");
+    }
+    // default - should never happen.
+    return "Apod, Cd74, ...";
+  }
+
   handleClick(g) {
     const { world, dispatch, userDefinedGenes } = this.props;
     const gene = g.target;
@@ -93,7 +114,7 @@ class GeneExpression extends React.Component {
       postUserErrorToast(
         "That's too many genes, you can have at most 15 user defined genes"
       );
-    } else if (!_.find(world.varAnnotations, { name: gene })) {
+    } else if (world.varAnnotations.col("name").indexOf(gene) === undefined) {
       postUserErrorToast("That doesn't appear to be a valid gene name.");
     } else {
       dispatch(actions.requestUserDefinedGene(gene));
@@ -116,9 +137,13 @@ class GeneExpression extends React.Component {
       const genes = _.pull(_.uniq(bulkAdd.split(/[ ,]+/)), "");
 
       genes.forEach(gene => {
-        if (userDefinedGenes.indexOf(gene) !== -1) {
+        if (gene.length === 0) {
+          keepAroundErrorToast("Must enter a gene name.");
+        } else if (userDefinedGenes.indexOf(gene) !== -1) {
           keepAroundErrorToast("That gene already exists");
-        } else if (!_.find(world.varAnnotations, { name: gene })) {
+        } else if (
+          world.varAnnotations.col("name").indexOf(gene) === undefined
+        ) {
           keepAroundErrorToast(
             `${gene} doesn't appear to be a valid gene name.`
           );
@@ -207,6 +232,7 @@ class GeneExpression extends React.Component {
                   /* this happens on 'enter' */
                   this.handleClick(g);
                 }}
+                inputProps={{ "data-testid": "gene-search" }}
                 inputValueRenderer={g => {
                   return "";
                 }}
@@ -214,13 +240,14 @@ class GeneExpression extends React.Component {
                 itemRenderer={renderGene.bind(this)}
                 items={
                   world && world.varAnnotations
-                    ? world.varAnnotations
-                    : [{ name: "No genes" }]
+                    ? world.varAnnotations.col("name").asArray()
+                    : ["No genes"]
                 }
                 popoverProps={{ minimal: true }}
               />
               <Button
                 className="bp3-button bp3-intent-primary"
+                data-testid={"add-gene"}
                 loading={userDefinedGenesLoading}
               >
                 Add
@@ -245,7 +272,7 @@ class GeneExpression extends React.Component {
                         this.setState({ bulkAdd: e.target.value });
                       }}
                       id="text-input-bulk-add"
-                      placeholder="Apod, Cd74, ..."
+                      placeholder={this.placeholderGeneNames()}
                       value={bulkAdd}
                     />
                     <Button
@@ -262,16 +289,17 @@ class GeneExpression extends React.Component {
           ) : null}
           {world && userDefinedGenes.length > 0
             ? _.map(userDefinedGenes, (geneName, index) => {
-                const values = world.varDataCache[geneName];
+                const values = world.varData.col(geneName);
                 if (!values) {
                   return null;
                 }
+                const summary = values.summarize();
                 return (
                   <HistogramBrush
                     key={geneName}
                     field={geneName}
                     zebra={index % 2 === 0}
-                    ranges={finiteExtent(values)}
+                    ranges={summary}
                     isUserDefined
                   />
                 );
@@ -290,18 +318,18 @@ class GeneExpression extends React.Component {
           <ExpressionButtons />
           {differential.diffExp
             ? _.map(differential.diffExp, (value, index) => {
-                const annotations = world.varAnnotations[value[0]];
-                const { name } = annotations;
-                const values = world.varDataCache[name];
+                const name = world.varAnnotations.at(value[0], "name");
+                const values = world.varData.col(name);
                 if (!values) {
                   return null;
                 }
+                const summary = values.summarize();
                 return (
                   <HistogramBrush
                     key={name}
                     field={name}
                     zebra={index % 2 === 0}
-                    ranges={finiteExtent(values)}
+                    ranges={summary}
                     isDiffExp
                     logFoldChange={value[1]}
                     pval={value[2]}

client/src/components/graph/graph.js

@@ -35,7 +35,8 @@ class Graph extends React.Component {
     this.graphPaddingRight = globals.leftSidebarWidth;
     this.renderCache = {
       positions: null,
-      colors: null
+      colors: null,
+      sizes: null
     };
     this.state = {
       svg: null,
@@ -83,12 +84,13 @@ class Graph extends React.Component {
   }
 
   componentDidUpdate(prevProps) {
+    const { renderCache } = this;
     const {
       world,
       crossfilter,
-      selectionUpdate,
       colorRGB,
-      responsive
+      responsive,
+      selectionUpdate
     } = this.props;
     const {
       reglRender,
@@ -109,35 +111,27 @@ class Graph extends React.Component {
 
     if (regl && world) {
       /* update the regl state */
-      const { obsLayout } = world;
-      const cellCount = crossfilter.size();
+      const { obsLayout, nObs } = world;
+      const X = obsLayout.col("X").asArray();
+      const Y = obsLayout.col("Y").asArray();
 
       // X/Y positions for each point - a cached value that only
       // changes if we have loaded entirely new cell data
       //
-      if (
-        !this.renderCache.positions ||
-        selectionUpdate !== prevProps.selectionUpdate
-      ) {
-        if (!this.renderCache.positions) {
-          this.renderCache.positions = new Float32Array(2 * cellCount);
-        }
+      if (!renderCache.positions || world !== prevProps.world) {
+        renderCache.positions = new Float32Array(2 * nObs);
 
         const glScaleX = scaleLinear([0, 1], [-1, 1]);
         const glScaleY = scaleLinear([0, 1], [1, -1]);
 
-        const offset = [d3.mean(obsLayout.X) - 0.5, d3.mean(obsLayout.Y) - 0.5];
+        const offset = [d3.mean(X) - 0.5, d3.mean(Y) - 0.5];
 
-        for (
-          let i = 0, { positions } = this.renderCache;
-          i < cellCount;
-          i += 1
-        ) {
-          positions[2 * i] = glScaleX(obsLayout.X[i] - offset[0]);
-          positions[2 * i + 1] = glScaleY(obsLayout.Y[i] - offset[1]);
+        for (let i = 0, { positions } = renderCache; i < nObs; i += 1) {
+          positions[2 * i] = glScaleX(X[i] - offset[0]);
+          positions[2 * i + 1] = glScaleY(Y[i] - offset[1]);
         }
         pointBuffer({
-          data: this.renderCache.positions,
+          data: renderCache.positions,
           dimension: 2
         });
 
@@ -152,30 +146,28 @@ class Graph extends React.Component {
       // could have changed for some other reason, but for now color is
       // the only metadata that changes client-side.  If this is problematic,
       // we could add some sort of color-specific indicator to the app state.
-      if (!this.renderCache.colors || colorRGB !== prevProps.colorRGB) {
+      if (!renderCache.colors || colorRGB !== prevProps.colorRGB) {
         const rgb = colorRGB;
-        if (!this.renderCache.colors) {
-          this.renderCache.colors = new Float32Array(3 * rgb.length);
+        if (!renderCache.colors) {
+          renderCache.colors = new Float32Array(3 * rgb.length);
         }
-        for (let i = 0, { colors } = this.renderCache; i < rgb.length; i += 1) {
+        for (let i = 0, { colors } = renderCache; i < rgb.length; i += 1) {
           colors.set(rgb[i], 3 * i);
         }
-        colorBuffer({ data: this.renderCache.colors, dimension: 3 });
+        colorBuffer({ data: renderCache.colors, dimension: 3 });
       }
 
-      // Sizes for each point - this is presumed to change each time the
-      // component receives new props.   Almost always a true assumption, as
-      // most property upates are due to changes driving a crossfilter
-      // selection set change.
-      //
-      if (!this.renderCache.sizes) {
-        this.renderCache.sizes = new Float32Array(cellCount);
+      // Sizes for each point - updates are triggered only when selected
+      // obs change
+      if (!renderCache.sizes || selectionUpdate !== prevProps.selectionUpdate) {
+        if (!renderCache.sizes) {
+          renderCache.sizes = new Float32Array(nObs);
+        }
+        crossfilter.fillByIsFiltered(renderCache.sizes, 4, 0.2);
+        sizeBuffer({ data: renderCache.sizes, dimension: 1 });
       }
 
-      crossfilter.fillByIsFiltered(this.renderCache.sizes, 4, 0.2);
-      sizeBuffer({ data: this.renderCache.sizes, dimension: 1 });
-
-      this.count = cellCount;
+      this.count = nObs;
 
       regl._refresh();
       this.reglDraw(
@@ -485,6 +477,7 @@ class Graph extends React.Component {
             <canvas
               width={responsive.width - this.graphPaddingRight}
               height={responsive.height - this.graphPaddingTop}
+              data-testid="layout"
               ref={canvas => {
                 this.reglCanvas = canvas;
               }}

client/src/components/graph/setupSVGandBrush.js

@@ -20,6 +20,7 @@ export default (
   const svg = d3
     .select("#graphAttachPoint")
     .append("svg")
+    .attr("data-testid", "layout-overlay")
     .attr("width", responsive.width - graphPaddingRight)
     .attr("height", responsive.height)
     .attr("class", `${styles.graphSVG}`);

client/src/components/leftsidebar.js

@@ -40,6 +40,7 @@ class LeftSideBar extends React.Component {
         }}
       >
         <p
+          data-testid="header"
           style={{
             position: "fixed",
             top: globals.cellxgeneTitleTopPadding,

client/src/components/scatterplot/scatterplot.js

@@ -19,7 +19,6 @@ import _drawPoints from "./drawPointsRegl";
 import scaleLinear from "../../util/scaleLinear";
 
 import { margin, width, height } from "./util";
-import { kvCache } from "../../util/stateManager";
 import finiteExtent from "../../util/finiteExtent";
 
 @connect(state => {
@@ -30,12 +29,16 @@ import finiteExtent from "../../util/finiteExtent";
     scatterplotYYaccessor
   } = state.controls;
   const expressionX =
-    world && scatterplotXXaccessor
-      ? kvCache.get(world.varDataCache, scatterplotXXaccessor)
+    world &&
+    scatterplotXXaccessor &&
+    world.varData.hasCol(scatterplotXXaccessor)
+      ? world.varData.col(scatterplotXXaccessor).asArray()
       : null;
   const expressionY =
-    world && scatterplotYYaccessor
-      ? kvCache.get(world.varDataCache, scatterplotYYaccessor)
+    world &&
+    scatterplotYYaccessor &&
+    world.varData.hasCol(scatterplotYYaccessor)
+      ? world.varData.col(scatterplotYYaccessor).asArray()
       : null;
 
   return {
@@ -65,12 +68,17 @@ class Scatterplot extends React.Component {
     super(props);
     this.count = 0;
     this.axes = false;
-    this.state = {
-      svg: null,
-      minimized: null,
+    this.renderCache = {
+      positions: null,
+      colors: null,
+      sizes: null,
       xScale: null,
       yScale: null
     };
+    this.state = {
+      svg: null,
+      minimized: null
+    };
   }
 
   componentDidMount() {
@@ -81,6 +89,7 @@ class Scatterplot extends React.Component {
     if (svg && expressionX && expressionY) {
       scales = Scatterplot.setupScales(expressionX, expressionY);
       this.drawAxesSVG(scales.xScale, scales.yScale, svg);
+      this.renderCache = { ...this.renderCache, ...scales };
     }
 
     const camera = _camera(this.reglCanvas, { scale: true, rotate: false });
@@ -113,8 +122,6 @@ class Scatterplot extends React.Component {
       pointBuffer,
       colorBuffer,
       svg,
-      xScale: scales ? scales.xScale : null,
-      yScale: scales ? scales.yScale : null,
       reglRender,
       camera,
       drawPoints
@@ -129,12 +136,11 @@ class Scatterplot extends React.Component {
       scatterplotYYaccessor,
       expressionX,
       expressionY,
-      colorRGB
+      colorRGB,
+      selectionUpdate
     } = this.props;
     const {
       reglRender,
-      xScale,
-      yScale,
       regl,
       pointBuffer,
       colorBuffer,
@@ -145,17 +151,12 @@ class Scatterplot extends React.Component {
     } = this.state;
 
     if (
-      world &&
-      svg &&
-      xScale &&
-      yScale &&
-      scatterplotXXaccessor &&
-      scatterplotYYaccessor &&
-      (scatterplotXXaccessor !== prevProps.scatterplotXXaccessor || // was CLU now FTH1 etc
-      scatterplotYYaccessor !== prevProps.scatterplotYYaccessor || // was CLU now FTH1 etc
-        !this.axes) // clicked off the tab and back again, rerender
+      scatterplotXXaccessor !== prevProps.scatterplotXXaccessor || // was CLU now FTH1 etc
+      scatterplotYYaccessor !== prevProps.scatterplotYYaccessor // was CLU now FTH1 etc
     ) {
-      this.drawAxesSVG(xScale, yScale, svg);
+      const scales = Scatterplot.setupScales(expressionX, expressionY);
+      this.drawAxesSVG(scales.xScale, scales.yScale, svg);
+      this.renderCache = { ...this.renderCache, ...scales };
     }
 
     if (reglRender && this.reglRenderState === "rendering") {
@@ -172,35 +173,51 @@ class Scatterplot extends React.Component {
       expressionX &&
       expressionY &&
       scatterplotXXaccessor &&
-      scatterplotYYaccessor &&
-      xScale &&
-      yScale
+      scatterplotYYaccessor
     ) {
+      const { renderCache } = this;
+      const { xScale, yScale } = this.renderCache;
       const cellCount = expressionX.length;
-      const positionsBuf = new Float32Array(2 * cellCount);
-      const colorsBuf = new Float32Array(3 * cellCount);
-      const sizesBuf = new Float32Array(cellCount);
 
-      const glScaleX = scaleLinear([0, width], [-0.95, 0.95]);
-      const glScaleY = scaleLinear([0, height], [-1, 1]);
-
-      /*
-        Construct Vectors
-      */
-      for (let i = 0; i < cellCount; i += 1) {
-        positionsBuf[2 * i] = glScaleX(xScale(expressionX[i]));
-        positionsBuf[2 * i + 1] = glScaleY(yScale(expressionY[i]));
+      // Points change when expressionX or expressionY change.
+      if (
+        !renderCache.positions ||
+        expressionX !== prevProps.expressionX ||
+        expressionY !== prevProps.expressionY
+      ) {
+        if (!renderCache.positions) {
+          renderCache.positions = new Float32Array(2 * cellCount);
+        }
+        const glScaleX = scaleLinear([0, width], [-0.95, 0.95]);
+        const glScaleY = scaleLinear([0, height], [-1, 1]);
+        for (let i = 0, { positions } = renderCache; i < cellCount; i += 1) {
+          positions[2 * i] = glScaleX(xScale(expressionX[i]));
+          positions[2 * i + 1] = glScaleY(yScale(expressionY[i]));
+        }
+        pointBuffer({ data: renderCache.positions, dimension: 2 });
       }
 
-      for (let i = 0; i < cellCount; i += 1) {
-        colorsBuf.set(colorRGB[i], 3 * i);
+      // Colors for each point - change only when props.colorsRGB change.
+      if (!renderCache.colors || colorRGB !== prevProps.colorRGB) {
+        if (!renderCache.colors) {
+          renderCache.colors = new Float32Array(3 * cellCount);
+        }
+        for (let i = 0, { colors } = renderCache; i < cellCount; i += 1) {
+          colors.set(colorRGB[i], 3 * i);
+        }
+        colorBuffer({ data: renderCache.colors, dimension: 3 });
       }
 
-      crossfilter.fillByIsFiltered(sizesBuf, 4, 0.2);
+      // Sizes for each point - updates are triggered only when selected
+      // obs change
+      if (!renderCache.sizes || selectionUpdate !== prevProps.selctionUpdate) {
+        if (!renderCache.sizes) {
+          renderCache.sizes = new Float32Array(cellCount);
+        }
+        crossfilter.fillByIsFiltered(renderCache.sizes, 4, 0.2);
+        sizeBuffer({ data: renderCache.sizes, dimension: 1 });
+      }
 
-      pointBuffer({ data: positionsBuf, dimension: 2 });
-      colorBuffer({ data: colorsBuf, dimension: 3 });
-      sizeBuffer({ data: sizesBuf, dimension: 1 });
       this.count = cellCount;
 
       regl._refresh();
@@ -213,16 +230,6 @@ class Scatterplot extends React.Component {
         camera
       );
     }
-
-    if (
-      expressionX &&
-      expressionY &&
-      (scatterplotXXaccessor !== prevProps.scatterplotXXaccessor || // was CLU now FTH1 etc
-        scatterplotYYaccessor !== prevProps.scatterplotYYaccessor)
-    ) {
-      const scales = Scatterplot.setupScales(expressionX, expressionY);
-      this.setState(scales);
-    }
   }
 
   static setupScales(expressionX, expressionY) {

client/src/middleware/updateCellColors.js

@@ -41,13 +41,13 @@ const updateCellColorsMiddleware = store => next => action => {
     action.type === "color by continuous metadata" ||
     action.type === "color by categorical metadata";
 
-  if (!filterJustChanged || !s.controls.world.obsAnnotations) {
+  const obsAnnotations = _.get(s.controls, "world.obsAnnotations", null);
+  if (!filterJustChanged || !obsAnnotations) {
     return next(
       action
     ); /* if the cells haven't loaded or the action wasn't a color change, bail */
   }
 
-  const { obsAnnotations } = s.controls.world;
   let colorScale;
   const colorsByRGB = new Array(obsAnnotations.length);
 
@@ -73,16 +73,16 @@ const updateCellColorsMiddleware = store => next => action => {
     });
 
     const key = action.colorAccessor;
+    const col = obsAnnotations.col(key).asArray();
     for (let i = 0, len = obsAnnotations.length; i < len; i += 1) {
-      const obs = obsAnnotations[i];
-      const cat = obs[key];
+      const cat = col[i];
       colorsByRGB[i] = colors[cat];
     }
   }
 
   if (action.type === "color by continuous metadata") {
     const colorBins = 100;
-    const [min, max] = [0, action.rangeMaxForColorAccessor];
+    const { min, max } = action.rangeForColorAccessor;
     colorScale = d3
       .scaleQuantile()
       .domain([min, max])
@@ -96,8 +96,9 @@ const updateCellColorsMiddleware = store => next => action => {
 
     const key = action.colorAccessor;
     const nonFiniteColor = parseRGB(globals.nonFiniteCellColor);
+    const col = obsAnnotations.col(key).asArray();
     for (let i = 0, len = obsAnnotations.length; i < len; i += 1) {
-      const val = obsAnnotations[i][key];
+      const val = col[i];
       if (Number.isFinite(val)) {
         const c = colorScale(val);
         colorsByRGB[i] = colors[c];

client/src/reducers/controls.js

@@ -1,9 +1,8 @@
 // jshint esversion: 6
 
 import _ from "lodash";
-import { polygonContains } from "d3";
 
-import { World, kvCache, WorldUtil } from "../util/stateManager";
+import { World, WorldUtil, ControlsHelper } from "../util/stateManager";
 import parseRGB from "../util/parseRGB";
 import Crossfilter from "../util/typedCrossfilter";
 import * as globals from "../globals";
@@ -14,91 +13,6 @@ import {
   diffexpDimensionName,
   makeContinuousDimensionName
 } from "../util/nameCreators";
-import { fillRange } from "../util/typedCrossfilter/util";
-
-/*
-Selection state for categoricals are tracked in an Object that
-has two main components for each category:
-1. mapping of option value to an index
-2. array of bool selection state by index
-Remember that option values can be ANY js type, except undefined/null.
-
-  {
-    _category_name_1: {
-      // map of option value to index
-      categoryIndices: Map([
-        catval1: index,
-        ...
-      ])
-
-      // index->selection true/false state
-      categorySelected: [ true/false, true/false, ... ]
-
-      // number of options
-      numCategories: number,
-
-      // isTruncated - true if the options for selection has
-      // been truncated (ie, was too large to implement)
-    }
-  }
-*/
-function topNCategories(summary) {
-  const counts = _.map(summary.categories, cat =>
-    summary.categoryCounts.get(cat)
-  );
-  const sortIndex = fillRange(new Array(summary.numCategories)).sort(
-    (a, b) => counts[b] - counts[a]
-  );
-  const sortedCategories = _.map(sortIndex, i => summary.categories[i]);
-  const sortedCounts = _.map(sortIndex, i => counts[i]);
-  const N = globals.maxCategoricalOptionsToDisplay;
-
-  if (sortedCategories.length < N) {
-    return [sortedCategories, sortedCounts];
-  }
-  return [sortedCategories.slice(0, N), sortedCounts.slice(0, N)];
-}
-
-function createCategoricalSelectionState(state, world) {
-  const res = {};
-  _.forEach(world.summary.obs, (value, key) => {
-    if (value.categories) {
-      const isColorField = key.includes("color") || key.includes("Color");
-      const isSelectableCategory =
-        !isColorField &&
-        key !== "name" &&
-        value.categories.length < state.maxCategoryItems;
-      if (isSelectableCategory) {
-        const [categoryValues, categoryCounts] = topNCategories(value);
-        const categoryIndices = new Map(categoryValues.map((v, i) => [v, i]));
-        const numCategories = categoryIndices.size;
-        const categorySelected = new Array(numCategories).fill(true);
-        const isTruncated = categoryValues.length < value.numCategories;
-        res[key] = {
-          categoryValues, // array: of natively typed category values
-          categoryIndices, // map: category value (native type) -> category index
-          categorySelected, // array: t/f selection state
-          numCategories, // number: of categories
-          isTruncated, // bool: true if list was truncated
-          categoryCounts // array: cardinality of each category
-        };
-      }
-    }
-  });
-  return res;
-}
-
-/*
-given a categoricalSelectionState, return the list of all category values
-where selection state is true (ie, they are selected).
-*/
-function selectedValuesForCategory(categorySelectionState) {
-  const selectedValues = _([...categorySelectionState.categoryIndices])
-    .filter(tuple => categorySelectionState.categorySelected[tuple[1]])
-    .map(tuple => tuple[0])
-    .value();
-  return selectedValues;
-}
 
 const Controls = (
   state = {
@@ -111,6 +25,7 @@ const Controls = (
 
     // the whole big bang
     universe: null,
+    fullUniverseCache: null,
 
     // all of the data + selection state
     world: null,
@@ -164,16 +79,14 @@ const Controls = (
     case "initial data load start": {
       return { ...state, loading: true };
     }
-    case "initial data load complete (universe exists)":
-    case "reset World to eq Universe": {
-      const { userDefinedGenes, diffexpGenes } = state;
+    case "initial data load complete (universe exists)": {
       /* first light - create world & other data-driven defaults */
       const { universe } = action;
       const world = World.createWorldFromEntireUniverse(universe);
       const colorRGB = new Array(universe.nObs).fill(
         parseRGB(globals.defaultCellColor)
       );
-      const categoricalSelectionState = createCategoricalSelectionState(
+      const categoricalSelectionState = ControlsHelper.createCategoricalSelectionState(
         state,
         world
       );
@@ -181,51 +94,58 @@ const Controls = (
       const dimensionMap = World.createObsDimensionMap(crossfilter, world);
       WorldUtil.clearCaches();
 
-      const worldVarDataCache = world.varDataCache;
-
-      // dimensionMap = {
-      //     layout_X: dim-for-X,
-      //     obsAnno_name: dim for an annotation,
-      //     varData_userDefined_genename: dim for user defined expression,
-      //     varData_diffexp_genename: dim for diff-exp added gene expression
-      // }
-      /* var dimensions */
-      if (userDefinedGenes.length > 0) {
-        /*
-          verbose & slightly confusing that we also access this as an object
-          in controls rather than an array, should be abstracted into
-          util ie., createDimensionsFromBothListsOfGenes(userGenes, diffExp)
-        */
-        _.forEach(userDefinedGenes, gene => {
-          dimensionMap[
-            userDefinedDimensionName(gene)
-          ] = World.createVarDimension(
-            /* "__var__" + */
-            world,
-            worldVarDataCache,
-            crossfilter,
-            gene
-          );
-        });
-      }
-
-      if (diffexpGenes.length > 0) {
-        _.forEach(diffexpGenes, gene => {
-          dimensionMap[diffexpDimensionName(gene)] = World.createVarDimension(
-            /* "__var__" + */
-            world,
-            worldVarDataCache,
-            crossfilter,
-            gene
-          );
-        });
-      }
-
       return {
         ...state,
         loading: false,
         error: null,
         universe,
+        fullUniverseCache: { world, crossfilter, dimensionMap },
+        world,
+        colorRGB,
+        categoricalSelectionState,
+        crossfilter,
+        dimensionMap,
+        colorAccessor: null,
+        resettingInterface: false
+      };
+    }
+    case "reset World to eq Universe": {
+      const {
+        userDefinedGenes,
+        diffexpGenes,
+        universe,
+        fullUniverseCache
+      } = state;
+      const { world, crossfilter } = fullUniverseCache;
+      // reset all crossfilter dimensions
+      _.forEach(fullUniverseCache.dimensionMap, dim => dim.filterAll());
+      const colorRGB = new Array(universe.nObs).fill(
+        parseRGB(globals.defaultCellColor)
+      );
+      const categoricalSelectionState = ControlsHelper.createCategoricalSelectionState(
+        state,
+        world
+      );
+
+      /* free dimensions not in cache (otherwise they leak) */
+      _.forEach(state.dimensionMap, (dim, dimName) => {
+        if (!fullUniverseCache.dimensionMap[dimName]) {
+          dim.dispose();
+        }
+      });
+      const dimensionMap = {
+        ...fullUniverseCache.dimensionMap,
+        ...ControlsHelper.createGenesDimMap(
+          userDefinedGenes,
+          diffexpGenes,
+          world,
+          crossfilter
+        )
+      };
+      WorldUtil.clearCaches();
+
+      return {
+        ...state,
         world,
         colorRGB,
         categoricalSelectionState,
@@ -247,48 +167,22 @@ const Controls = (
       const colorRGB = new Array(world.nObs).fill(
         parseRGB(globals.defaultCellColor)
       );
-      const categoricalSelectionState = createCategoricalSelectionState(
+      const categoricalSelectionState = ControlsHelper.createCategoricalSelectionState(
         state,
         world
       );
       const crossfilter = Crossfilter(world.obsAnnotations);
-      const dimensionMap = World.createObsDimensionMap(crossfilter, world);
+      const dimensionMap = {
+        ...World.createObsDimensionMap(crossfilter, world),
+        ...ControlsHelper.createGenesDimMap(
+          userDefinedGenes,
+          diffexpGenes,
+          world,
+          crossfilter
+        )
+      };
       WorldUtil.clearCaches();
 
-      const worldVarDataCache = world.varDataCache;
-      /* var dimensions */
-
-      if (userDefinedGenes.length > 0) {
-        /*
-          verbose & slightly confusing that we also access this as an object
-          in controls rather than an array, should be abstracted into
-          util ie., createDimensionsFromBothListsOfGenes(userGenes, diffExp)
-        */
-        _.forEach(userDefinedGenes, gene => {
-          dimensionMap[
-            userDefinedDimensionName(gene)
-          ] = World.createVarDimension(
-            /* "__var__" + */
-            world,
-            worldVarDataCache,
-            crossfilter,
-            gene
-          );
-        });
-      }
-
-      if (diffexpGenes.length > 0) {
-        _.forEach(diffexpGenes, gene => {
-          dimensionMap[diffexpDimensionName(gene)] = World.createVarDimension(
-            /* "__var__" + */
-            world,
-            worldVarDataCache,
-            crossfilter,
-            gene
-          );
-        });
-      }
-
       return {
         ...state,
         loading: false,
@@ -303,28 +197,66 @@ const Controls = (
     }
     case "expression load success": {
       const { world, universe } = state;
-      let universeVarDataCache = universe.varDataCache;
-      let worldVarDataCache = world.varDataCache;
+      let universeVarData = universe.varData;
+      let worldVarData = world.varData;
+
+      // Load new expression data into the varData dataframes, if
+      // not already present.
       _.forEach(action.expressionData, (val, key) => {
-        universeVarDataCache = kvCache.set(universeVarDataCache, key, val);
-        if (kvCache.get(worldVarDataCache, key) === undefined) {
-          worldVarDataCache = kvCache.set(
-            worldVarDataCache,
+        // If not already in universe.varData, save entire expression column
+        if (!universeVarData.hasCol(key)) {
+          universeVarData = universeVarData.withCol(key, val);
+        }
+
+        // If not already in world.varData, save sliced expression column
+        if (!worldVarData.hasCol(key)) {
+          // Slice if world !== universe, else just use whole column.
+          // Use the obsAnnotation index as the cut key, as we keep
+          // all world dataframes in sync.
+          let worldValSlice = val;
+          if (!World.worldEqUniverse(world, universe)) {
+            worldValSlice = universeVarData
+              .subset(world.obsAnnotations.rowIndex.keys(), [key], null)
+              .icol(0)
+              .asArray();
+          }
+
+          // Now build world's varData dataframe
+          worldVarData = worldVarData.withCol(
             key,
-            World.subsetVarData(world, universe, val)
+            worldValSlice,
+            world.obsAnnotations.rowIndex
           );
         }
       });
 
+      // Prune size of varData "cache" if getting out of hand....
+      const { userDefinedGenes, diffexpGenes } = state;
+      const allTheGenesWeNeed = _.uniq(
+        [].concat(
+          userDefinedGenes,
+          diffexpGenes,
+          Object.keys(action.expressionData)
+        )
+      );
+      universeVarData = ControlsHelper.pruneVarDataCache(
+        universeVarData,
+        allTheGenesWeNeed
+      );
+      worldVarData = ControlsHelper.pruneVarDataCache(
+        worldVarData,
+        allTheGenesWeNeed
+      );
+
       return {
         ...state,
         universe: {
           ...universe,
-          varDataCache: universeVarDataCache
+          varData: universeVarData
         },
         world: {
           ...world,
-          varDataCache: worldVarDataCache
+          varData: worldVarData
         }
       };
     }
@@ -342,17 +274,12 @@ const Controls = (
     }
     case "request user defined gene success": {
       const { world, crossfilter, dimensionMap, userDefinedGenes } = state;
-      const worldVarDataCache = world.varDataCache;
       const _userDefinedGenes = userDefinedGenes.slice();
       const gene = action.data.genes[0];
 
-      dimensionMap[userDefinedDimensionName(gene)] = World.createVarDimension(
-        /* "__var__" + */
-        world,
-        worldVarDataCache,
-        crossfilter,
-        gene
-      );
+      dimensionMap[
+        userDefinedDimensionName(gene)
+      ] = World.createVarDataDimension(world, crossfilter, gene);
 
       return {
         ...state,
@@ -363,18 +290,15 @@ const Controls = (
     }
     case "request differential expression success": {
       const { world, crossfilter, dimensionMap } = state;
-      const worldVarDataCache = world.varDataCache;
       const _diffexpGenes = [];
 
       action.data.forEach(d => {
-        _diffexpGenes.push(world.varAnnotations[d[0]].name);
+        _diffexpGenes.push(world.varAnnotations.at(d[0], "name"));
       });
 
       _.forEach(_diffexpGenes, gene => {
-        dimensionMap[diffexpDimensionName(gene)] = World.createVarDimension(
-          /* "__var__" + */
+        dimensionMap[diffexpDimensionName(gene)] = World.createVarDataDimension(
           world,
-          worldVarDataCache,
           crossfilter,
           gene
         );
@@ -387,13 +311,10 @@ const Controls = (
       };
     }
     case "clear differential expression": {
-      const { world, universe, dimensionMap } = state;
+      const { world, dimensionMap } = state;
       const _dimensionMap = dimensionMap;
-      const universeVarDataCache = universe.varDataCache;
-      const worldVarDataCache = world.varDataCache;
-
       _.forEach(action.diffExp, values => {
-        const { name } = world.varAnnotations[values[0]];
+        const name = world.varAnnotations.at(values[0], "name");
         // clean up crossfilter dimensions
         const dimension = dimensionMap[diffexpDimensionName(name)];
         dimension.dispose();
@@ -402,15 +323,7 @@ const Controls = (
       return {
         ...state,
         dimensionMap: _dimensionMap,
-        diffexpGenes: [],
-        universe: {
-          ...universe,
-          varDataCache: universeVarDataCache
-        },
-        world: {
-          ...world,
-          varDataCache: worldVarDataCache
-        }
+        diffexpGenes: []
       };
     }
     case "user defined gene": {
@@ -563,7 +476,7 @@ const Controls = (
       // update the filter to match all selected options
       const cat = newCategoricalSelectionState[action.metadataField];
       state.dimensionMap[obsAnnoDimensionName(action.metadataField)].filterEnum(
-        selectedValuesForCategory(cat)
+        ControlsHelper.selectedValuesForCategory(cat)
       );
 
       return {
@@ -587,7 +500,7 @@ const Controls = (
       // update the filter to match all selected options
       const cat = newCategoricalSelectionState[action.metadataField];
       state.dimensionMap[obsAnnoDimensionName(action.metadataField)].filterEnum(
-        selectedValuesForCategory(cat)
+        ControlsHelper.selectedValuesForCategory(cat)
       );
 
       return {

client/src/util/dataframe/dataframe.js

@@ -0,0 +1,574 @@
+import { IdentityInt32Index, isLabelIndex } from "./labelIndex";
+// weird cross-dependency that we should clean up someday...
+import { sort } from "../typedCrossfilter/sort";
+import { isTypedArray, isArrayOrTypedArray, callOnceLazy } from "./util";
+import { summarizeContinuous, summarizeCategorical } from "./summarize";
+
+/*
+Dataframe is an immutable 2D matrix similiar to Python Pandas Dataframe,
+but (currently) without all of the surrounding support functions.
+Data is stored in column-major layout, and each column is monomorphic.
+
+It supports:
+* Relatively efficient creation, cloning and subsetting
+* Very efficient columnar access (eg, sum down a column), and access
+  to the underlying column arrays.
+* Data access by row/col offset or label.  Labels are reasonably well
+  optimized for both numeric lables and arbitrary (eg, sting) labels.
+
+It does not currently support:
+* Views on matrix subset - for currently known access patterns,
+  it is more effiicent to copy on subsetting, optimizing for access
+  speed over memory use.
+* JS iterators - they are too slow.  Use explicit iteration over
+  offest or labels.
+
+Important assumptions embedded in the API:
+* Columns are implicitly categorical if they are a JS Array and numeric
+  (aka continuous) if they are a TypedArray.
+
+There are three index types for row/col indexing:
+* IdentityInt32Index - noop index, where the index label is the offset.
+* KeyIndex - index arbitrary JS objects.
+* DenseInt32Index - integer indexing.  Optimization over KeyIndex as it uses
+  Int32Array as a back-map to offsets.  This means that the index array
+  must be sized to [minLabel, maxLabel), so this is only useful when the label
+  range is relatively close the underlying offset range [minOffset, maxOffset).
+
+All private functions/methods/fields are prefixed by '__', eg, __compile().
+Don't use them outside of this file.
+
+Simple example:
+
+  // default indexing is integer offset.
+  const df = Dataframe.create([2,2], [['a', 'b'], [0, 1]])
+  console.log(df.at(0,0));  // outputs: a
+  console.log(df.col(1).asArray());   // outputs: [0, 1]
+
+  // KeyIndex
+  const df = new Dataframe([1,2], [['a'], ['b']], null, new KeyIndex(['A', 'B']))
+  console.log(df.at(0, 'A')); // outputs: a
+  console.log(df.col('A').asArray(); // outputs: ['a']
+
+Performance tuning is primarily focused on columnar access patterns, which is the
+dominant pattern in cellxgene.
+*/
+
+/**
+Dataframe
+**/
+
+class Dataframe {
+  /**
+  Constructors & factories
+  **/
+
+  constructor(dims, columnarData, rowIndex = null, colIndex = null) {
+    /*
+    The base constructor is relatively hard to use - as an alternative,
+    see factory methods and clone/slice, below.
+
+    Parameters:
+      * dims - 2D array describing intendend dimensionality: [nRows,nCols].
+      * columnarData - JS array, nCols in length, containing array
+        or TypedArray of length nRows.
+      * rowIndex/colIndex - null (create default index using offsets as key),
+        or a caller-provided index.
+    All columns and indices must have appropriate dimensionality.
+    */
+    const [nRows, nCols] = dims;
+    if (nRows < 0 || nCols < 0) {
+      throw new RangeError("Dataframe dimensions must be positive");
+    }
+    if (!rowIndex) {
+      rowIndex = new IdentityInt32Index(nRows);
+    }
+    if (!colIndex) {
+      colIndex = new IdentityInt32Index(nCols);
+    }
+    Dataframe.__errorChecks(dims, columnarData, rowIndex, colIndex);
+
+    this.__columns = Array.from(columnarData);
+    this.dims = dims;
+    this.length = nRows; // convenience accessor for row dimension
+    this.rowIndex = rowIndex;
+    this.colIndex = colIndex;
+
+    this.__compile();
+  }
+
+  static __errorChecks(dims, columnarData, rowIndex, colIndex) {
+    const [nRows, nCols] = dims;
+
+    /* check for expected types */
+    if (!Array.isArray(columnarData)) {
+      throw new TypeError("Dataframe constructor requires array of columns");
+    }
+    if (!columnarData.every(c => isArrayOrTypedArray(c))) {
+      throw new TypeError("Dataframe columns must all be Array or TypedArray");
+    }
+    if (!isLabelIndex(rowIndex)) {
+      throw new TypeError("Dataframe rowIndex is an unsupported type.");
+    }
+    if (!isLabelIndex(colIndex)) {
+      throw new TypeError("Dataframe colIndex is an unsupported type.");
+    }
+
+    /* check for expected dimensionality / size */
+    if (
+      nCols !== columnarData.length ||
+      !columnarData.every(c => c.length === nRows)
+    ) {
+      throw new RangeError(
+        "Dataframe dimension does not match provided data shape"
+      );
+    }
+    if (nRows !== rowIndex.size()) {
+      throw new RangeError(
+        "Dataframe rowIndex must have same size as underlying data"
+      );
+    }
+    if (nCols !== colIndex.size()) {
+      throw new RangeError(
+        "Dataframe colIndex must have same size as underlying data"
+      );
+    }
+  }
+
+  __compile() {
+    /*
+    Compile data accessors for each column.
+
+    Each column accessor is a function which will lookup data by
+    index (ie, is equivalent to dataframe.get(row, col), where 'col'
+    is fixed.
+
+    In addition, each column accessor has several functions:
+
+    asArray() -- return the entire column as a native Array or TypedArray.
+      Crucially, this native array only supports label indexing.
+      Example:
+        const arr = df.col('a').asArray();
+
+    has(rlabel) -- return boolean indicating of the row label
+      is contained within the column.  Example:
+        const isInColumn = df.col('a').includes(99)
+      For the default offset indexing, this is identical to:
+        const isInColumn = (99 > 0) && (99 < df.nRows);
+
+    ihas(roffset) -- same as has(), but accepts a row offset
+      instead of a row label.
+
+    indexOf(value) -- return the label (not offset) of the first instance of
+      'value' in the column.  If you want the offset, just use the builtin JS
+      indexOf() function, available on both Array and TypedArray.
+
+    iget(offset) -- return the value at 'offset'
+
+    */
+    const { getOffset, getLabel } = this.rowIndex;
+    this.__columnsAccessor = this.__columns.map(column => {
+      const { length } = column;
+
+      /* get value by row label */
+      const get = function get(rlabel) {
+        return column[getOffset(rlabel)];
+      };
+
+      /* get value by row offset */
+      const iget = function iget(roffset) {
+        return column[roffset];
+      };
+
+      /* full column array access */
+      const asArray = function asArray() {
+        return column;
+      };
+
+      /* test for row label inclusion in column */
+      const has = function has(rlabel) {
+        const offset = getOffset(rlabel);
+        return offset >= 0 && offset < length;
+      };
+
+      const ihas = function ihas(offset) {
+        return offset >= 0 && offset < length;
+      };
+
+      /*
+      return first label (index) at which the value is found in this column,
+      or undefined if not found.
+
+      NOTE: not found return is DIFFERENT than the default Array.indexOf as
+      -1 is a plausible Dataframe row/col label.
+      */
+      const indexOf = function indexOf(value) {
+        const offset = column.indexOf(value);
+        if (offset === -1) {
+          return undefined;
+        }
+        return getLabel(offset);
+      };
+
+      /*
+      Summarize the column data. Lazy eval;
+      */
+      const summarize = callOnceLazy(() =>
+        isTypedArray(column)
+          ? summarizeContinuous(column)
+          : summarizeCategorical(column)
+      );
+
+      get.summarize = summarize;
+      get.asArray = asArray;
+      get.has = has;
+      get.ihas = ihas;
+      get.indexOf = indexOf;
+      get.iget = iget;
+      return get;
+    });
+  }
+
+  clone() {
+    /*
+    Clone this dataframe
+    */
+    return new this.constructor(
+      this.dims,
+      [...this.__columns],
+      this.rowIndex,
+      this.colIndex
+    );
+  }
+
+  withCol(label, colData, withRowIndex = null) {
+    /*
+    Create a new DF, which is `this` plus the new column. Example:
+    const newDf = df.withCol("foo", [1,2,3]);
+
+    Dimensionality of new column must match existing dataframe.
+
+    Special case: empty dataframe will accept any size column.  Example:
+    const newDf = Dataframe.empty().withCol("foo", [1,2,3]);
+
+    If `withRowIndex` specified, the provided index will become the
+    rowIndex for the newly created dataframe.   If not specified,
+    the rowIndex from `this` will be used (ie, the rowIndex is
+    unchanged).
+    */
+    let dims;
+    let rowIndex;
+    if (this.isEmpty()) {
+      dims = [colData.length, 1];
+      rowIndex = null;
+    } else {
+      dims = [this.dims[0], this.dims[1] + 1];
+      ({ rowIndex } = this);
+    }
+
+    if (withRowIndex) {
+      rowIndex = withRowIndex;
+    }
+
+    const columns = [...this.__columns];
+    columns.push(colData);
+    const colIndex = this.colIndex.withLabel(label);
+    return new this.constructor(dims, columns, rowIndex, colIndex);
+  }
+
+  dropCol(label) {
+    /*
+    Create a new dataframe, omitting one columns.
+
+    const newDf = df.dropCol("colors");
+    */
+    const dims = [this.dims[0], this.dims[1] - 1];
+    const coffset = this.colIndex.getOffset(label);
+    const columns = [...this.__columns];
+    columns.splice(coffset, 1);
+    const colIndex = this.colIndex.dropLabel(label);
+    return new this.constructor(dims, columns, this.rowIndex, colIndex);
+  }
+
+  static empty(rowIndex = null, colIndex = null) {
+    return new Dataframe([0, 0], [], rowIndex, colIndex);
+  }
+
+  static create(dims, columnarData) {
+    /*
+    Create a dataframe from raw columnar data.  All column arrays
+    must have the same length.   Identity indexing will be used.
+
+    Example:
+    const df = Dataframe.create([2,2], [new Uint32Array(2), new Float32Array(2)]);
+    */
+    return new Dataframe(dims, columnarData, null, null);
+  }
+
+  __subset(rowOffsets, colOffsets, withRowIndex) {
+    const dims = [...this.dims];
+
+    const getSortedLabelAndOffsets = (offsets, index) => {
+      /*
+      Given offsets, return both offsets and associated lables,
+      sorted by offset.
+      */
+      if (!offsets) {
+        return [null, null];
+      }
+      const sortedOffsets = sort(offsets);
+      const sortedLabels = new Array(sortedOffsets.length);
+      for (let i = 0, l = sortedOffsets.length; i < l; i += 1) {
+        sortedLabels[i] = index.getLabel(sortedOffsets[i]);
+      }
+      return [sortedLabels, sortedOffsets];
+    };
+
+    let { colIndex } = this;
+    if (colOffsets) {
+      let colLabels;
+      [colLabels, colOffsets] = getSortedLabelAndOffsets(
+        colOffsets,
+        this.colIndex
+      );
+      dims[1] = colOffsets.length;
+      colIndex = this.colIndex.subsetLabels(colLabels);
+    }
+
+    let { rowIndex } = this;
+    if (withRowIndex) rowIndex = withRowIndex;
+    if (rowOffsets) {
+      let rowLabels;
+      [rowLabels, rowOffsets] = getSortedLabelAndOffsets(
+        rowOffsets,
+        this.rowIndex
+      );
+      dims[0] = rowLabels.length;
+      if (!withRowIndex) rowIndex = this.rowIndex.subsetLabels(rowLabels);
+    }
+
+    /* subset columns */
+    let columns = this.__columns;
+    if (colOffsets) {
+      columns = new Array(colOffsets.length);
+      for (let i = 0, l = colOffsets.length; i < l; i += 1) {
+        columns[i] = this.__columns[colOffsets[i]];
+      }
+    }
+
+    /* subset rows */
+    if (rowOffsets) {
+      columns = columns.map(col => {
+        const newCol = new col.constructor(rowOffsets.length);
+        for (let i = 0, l = rowOffsets.length; i < l; i += 1) {
+          newCol[i] = col[rowOffsets[i]];
+        }
+        return newCol;
+      });
+    }
+    return new Dataframe(dims, columns, rowIndex, colIndex);
+  }
+
+  subset(rowLabels, colLabels = null, withRowIndex = null) {
+    /*
+    Subset by row/col labels.
+
+    withRowIndex allows assignment of new row index during subset operation.
+    If withRowIndex === null, it will reset the index to identity (offset)
+    indexing.  if withRowIndex is a label index object, it will be used
+    for the new dataframe.
+    */
+    const toOffsets = (labels, index) => {
+      if (!labels) {
+        return null;
+      }
+      return labels.map(label => {
+        const off = index.getOffset(label);
+        if (off === undefined) {
+          throw new RangeError(`unknown label: ${label}`);
+        }
+        return off;
+      });
+    };
+
+    const rowOffsets = toOffsets(rowLabels, this.rowIndex);
+    const colOffsets = toOffsets(colLabels, this.colIndex);
+    return this.__subset(rowOffsets, colOffsets, withRowIndex);
+  }
+
+  isubset(rowOffsets, colOffsets = null, withRowIndex = null) {
+    /*
+    Subset by row/col offset.
+
+    withRowIndex allows assignment of new row index during subset operation.
+    If withRowIndex === null, it will reset the index to identity (offset)
+    indexing.  if withRowIndex is a label index object, it will be used
+    for the new dataframe.
+    */
+    return this.__subset(rowOffsets, colOffsets, withRowIndex);
+  }
+
+  isubsetMask(rowMask, colMask = null, withRowIndex = null) {
+    /*
+    Subset on row/column based upon a truthy/falsey array (a mask).
+
+    withRowIndex allows assignment of new row index during subset operation.
+    If withRowIndex === null, it will reset the index to identity (offset)
+    indexing.  if withRowIndex is a label index object, it will be used
+    for the new dataframe.
+    */
+    const [nRows, nCols] = this.dims;
+    if (
+      (rowMask && rowMask.length !== nRows) ||
+      (colMask && colMask.length !== nCols)
+    ) {
+      throw new RangeError("boolean arrays must match row/col dimensions");
+    }
+
+    /* convert masks to lists - method wastes space, but is fast */
+    const toList = (mask, maxSize) => {
+      if (!mask) {
+        return null;
+      }
+      const list = new Int32Array(maxSize);
+      let elems = 0;
+      for (let i = 0, l = mask.length; i < l; i += 1) {
+        if (mask[i]) {
+          list[elems] = i;
+          elems += 1;
+        }
+      }
+      return new Int32Array(list.buffer, 0, elems);
+    };
+    const rowOffsets = toList(rowMask, nRows);
+    const colOffsets = toList(colMask, nCols);
+    return this.__subset(rowOffsets, colOffsets, withRowIndex);
+  }
+
+  /**
+  Data access with row/col.
+  **/
+
+  col(columnLabel) {
+    /*
+    Return accessor bound to a column.  Allows random row access
+    based upon the row indexing.  Returns undefined if the
+    columnLabel is not present in the dataframe.
+
+    Example for a dataframe with string labeled columns, and
+    default (offset) indices for rows (eg, [0, 'foo'])
+
+      const getValue = df.col('foo');
+      for (let r = 0; r < df.nRows; r += 1) {
+        console.log(r, getValue(r));
+      }
+
+    See __compile() for the functions available in a column accessor.
+    */
+    const coff = this.colIndex.getOffset(columnLabel);
+    return this.__columnsAccessor[coff];
+  }
+
+  icol(columnOffset) {
+    /*
+    Return column accessor by offset.
+    */
+    return this.__columnsAccessor[columnOffset];
+  }
+
+  at(r, c) {
+    /*
+    Access a single value, for a row/col label pair.
+
+    For performance reasons, there are no bounds or existance
+    checks on labels, and no defined behavior when these are supplied.
+    May return undefined, throw an Error, or do something else for
+    non-existant labels.  If you want predictable out-of-bounds
+    behavior, use has(), eg,
+
+    const myVal = df.has(r,l) ? df.at(r,l) : undefined;
+    */
+    const coff = this.colIndex.getOffset(c);
+    const roff = this.rowIndex.getOffset(r);
+    return this.__columns[coff][roff];
+  }
+
+  iat(r, c) {
+    /*
+    Access a single value, for a row/col offset (integer) position.
+
+    For performance reasons, there are no bounds checks on row/col offsets
+    or other well-defined behavior for out-of-bounds values.  If you want
+    well-defined bounds checking, use ihas(), eg,
+
+    const myVal = df.ihas(r, c) ? df.iat(r, c) : undefined;
+    */
+    return this.__columns[c][r];
+  }
+
+  has(r, c) {
+    /*
+    Test if row/col labels exist in the dataframe - returns true/false
+    */
+    const [nRows, nCols] = this.dims;
+    const coff = this.colIndex.getOffset(c);
+    const roff = this.rowIndex.getOffset(r);
+    return coff >= 0 && coff < nCols && roff >= 0 && roff < nRows;
+  }
+
+  ihas(r, c) {
+    /*
+    Test if row/col offset (integer) position exists in the
+    dataframe - returns true/false
+    */
+    const [nRows, nCols] = this.dims;
+    return c >= 0 && c < nCols && r >= 0 && r < nRows;
+  }
+
+  hasCol(c) {
+    /*
+    Test if col label exists - return true/false
+    */
+    return !!this.col(c);
+  }
+
+  isEmpty() {
+    /*
+    Return true if this is an empty dataframe, ie, has dimensions [0,0]
+    */
+    const [rows, cols] = this.dims;
+    return rows === 0 && cols === 0;
+  }
+
+  /****
+  Functional (map/reduce/etc) data access
+
+  XXX: not yet implemented, as there is no clear use case.   Can easily
+  add these as useful.
+  ****/
+
+  /*
+  Map & reduce of column or row
+
+  XXX TODO remainder of map/reduce functions:  mapCol, mapRow, reduceRow, ...
+  */
+  /* comment out until we have a use for this
+
+  reduceCol(clabel, callback, initialValue) {
+    const coff = this.colIndex.getOffset(clabel);
+    const column = this.__columns[coff];
+    let start = 0;
+    let acc = initialValue;
+    if (initialValue === undefined) {
+      acc = column[0];
+      start = 1;
+    }
+    for (let i = start, l = column.length; i < l; i += 1) {
+      acc = callback(acc, column[i]);
+    }
+    return acc;
+  }
+  */
+}
+
+export default Dataframe;

client/src/util/dataframe/index.js

@@ -0,0 +1,2 @@
+export { default as Dataframe } from "./dataframe";
+export { DenseInt32Index, IdentityInt32Index, KeyIndex } from "./labelIndex";

client/src/util/dataframe/labelIndex.js

@@ -0,0 +1,244 @@
+/**
+Label indexing - map a label to & from an integer offset.  See Dataframe
+for how this is used.
+**/
+
+/*
+Private utility functions
+*/
+function extent(tarr) {
+  let min = 0x7fffffff;
+  let max = ~min; // eslint-disable-line no-bitwise
+  for (let i = 0, l = tarr.length; i < l; i += 1) {
+    const v = tarr[i];
+    if (v < min) {
+      min = v;
+    }
+    if (v > max) {
+      max = v;
+    }
+  }
+  return [min, max];
+}
+
+function fillRange(arr, start = 0) {
+  const larr = arr;
+  for (let i = 0, l = larr.length; i < l; i += 1) {
+    larr[i] = i + start;
+  }
+  return larr;
+}
+
+/* eslint-disable class-methods-use-this */
+class IdentityInt32Index {
+  /*
+  identity/noop index, with small assumptions that labels are int32
+  */
+  constructor(maxOffset) {
+    this.maxOffset = maxOffset;
+  }
+
+  keys() {
+    // memoize
+    const k = fillRange(new Int32Array(this.maxOffset));
+    this.keys = function keys() {
+      return k;
+    };
+    return k;
+  }
+
+  getOffset(i) {
+    // label to offset
+    return i;
+  }
+
+  getLabel(i) {
+    // offset to label
+    return i;
+  }
+
+  size() {
+    return this.maxOffset;
+  }
+
+  __promote(labelArray) {
+    /*
+    time/space decision - based on the resulting density
+    */
+    const [minLabel, maxLabel] = extent(labelArray);
+    const labelSpaceSize = maxLabel - minLabel + 1;
+    const density = labelSpaceSize / this.maxOffset;
+    /* 0.1 is a magic number, that needs testing to optimize */
+    if (density < 0.1) {
+      return new KeyIndex(labelArray);
+    }
+    return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
+  }
+
+  subsetLabels(labelArray) {
+    return this.__promote(labelArray);
+  }
+
+  withLabel(label) {
+    if (label === this.maxOffset) {
+      return new IdentityInt32Index(label + 1);
+    }
+    return this.__promote([...this.keys(), label]);
+  }
+
+  dropLabel(label) {
+    if (label === this.maxOffset - 1) {
+      return new IdentityInt32Index(label);
+    }
+    const labelArray = [...this.keys()];
+    labelArray.splice(labelArray.indexOf(label), 1);
+    return this.__promote(labelArray);
+  }
+}
+/* eslint-enable class-methods-use-this */
+
+/* eslint-disable class-methods-use-this */
+class DenseInt32Index {
+  /*
+  DenseInt32Index indexes integer labels, and uses Int32Array typed arrays
+  for both forward and reverse indexing.   This means that the min/max range
+  of the forward index labels must be known a priori (so that the index
+  array can be pre-allocated).
+  */
+  constructor(labels, labelRange = null) {
+    if (labels.constructor !== Int32Array) {
+      labels = new Int32Array(labels);
+    }
+
+    if (!labelRange) {
+      labelRange = extent(labels);
+    }
+    const [minLabel, maxLabel] = labelRange;
+    const labelSpaceSize = maxLabel - minLabel + 1;
+    const index = new Int32Array(labelSpaceSize).fill(-1);
+    for (let i = 0, l = labels.length; i < l; i += 1) {
+      const label = labels[i];
+      index[label - minLabel] = i;
+    }
+
+    this.minLabel = minLabel;
+    this.rindex = labels;
+    this.index = index;
+    this.__compile();
+  }
+
+  __compile() {
+    const { minLabel, index, rindex } = this;
+    this.getOffset = function getOffset(l) {
+      return index[l - minLabel];
+    };
+    this.getLabel = function getLabel(i) {
+      return rindex[i];
+    };
+  }
+
+  keys() {
+    return this.rindex;
+  }
+
+  size() {
+    return this.rindex.length;
+  }
+
+  __promote(labelArray) {
+    /*
+    time/space decision - if we are going to use less than 10% of the
+    dense index space, switch to a KeyIndex (which is slower, but uses
+    less memory for sparse label spaces).
+    */
+    const [minLabel, maxLabel] = extent(labelArray);
+    const labelSpaceSize = maxLabel - minLabel + 1;
+    const density = labelSpaceSize / this.rindex.length;
+    /* 0.1 is a magic number, that needs testing to optimize */
+    if (density < 0.1) {
+      return new KeyIndex(labelArray);
+    }
+    return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
+  }
+
+  subsetLabels(labelArray) {
+    return this.__promote(labelArray);
+  }
+
+  withLabel(label) {
+    return this.__promote([...this.keys(), label]);
+  }
+
+  dropLabel(label) {
+    const labelArray = [...this.keys()];
+    labelArray.splice(labelArray.indexOf(label), 1);
+    return this.__promote(labelArray);
+  }
+}
+/* eslint-enable class-methods-use-this */
+
+/* eslint-disable class-methods-use-this */
+class KeyIndex {
+  /*
+  KeyIndex indexes arbitrary JS primitive types, and uses a Map()
+  as its core data structure.
+  */
+  constructor(labels) {
+    const index = new Map();
+    if (labels === undefined) {
+      labels = [];
+    }
+    const rindex = labels;
+    labels.forEach((v, i) => {
+      index.set(v, i);
+    });
+
+    this.index = index;
+    this.rindex = rindex;
+    this.__compile();
+  }
+
+  __compile() {
+    const { index, rindex } = this;
+    this.getOffset = function getOffset(k) {
+      return index.get(k);
+    };
+    this.getLabel = function getLabel(i) {
+      return rindex[i];
+    };
+  }
+
+  keys() {
+    return this.rindex;
+  }
+
+  size() {
+    return this.rindex.length;
+  }
+
+  subsetLabels(labelArray) {
+    return new KeyIndex(labelArray);
+  }
+
+  withLabel(label) {
+    return new KeyIndex([...this.rindex, label]);
+  }
+
+  dropLabel(label) {
+    const idx = this.rindex.indexOf(label);
+    const labelArray = [...this.rindex];
+    labelArray.splice(idx, 1);
+    return new KeyIndex(labelArray);
+  }
+}
+/* eslint-enable class-methods-use-this */
+
+function isLabelIndex(i) {
+  return (
+    i instanceof IdentityInt32Index ||
+    i instanceof DenseInt32Index ||
+    i instanceof KeyIndex
+  );
+}
+
+export { DenseInt32Index, IdentityInt32Index, KeyIndex, isLabelIndex };

client/src/util/dataframe/summarize.js

@@ -0,0 +1,57 @@
+/*
+Private dataframe support functions
+*/
+
+export function summarizeContinuous(col) {
+  let min;
+  let max;
+  let nan = 0;
+  let pinf = 0;
+  let ninf = 0;
+  if (col) {
+    for (let r = 0, l = col.length; r < l; r += 1) {
+      const val = Number(col[r]);
+      if (Number.isFinite(val)) {
+        if (min === undefined) {
+          min = val;
+          max = val;
+        } else {
+          min = val < min ? val : min;
+          max = val > max ? val : max;
+        }
+      } else if (Number.isNaN(val)) {
+        nan += 1;
+      } else if (val > 0) {
+        pinf += 1;
+      } else {
+        ninf += 1;
+      }
+    }
+  }
+  return {
+    categorical: false,
+    min,
+    max,
+    nan,
+    pinf,
+    ninf
+  };
+}
+
+export function summarizeCategorical(col) {
+  const categoryCounts = new Map();
+  if (col) {
+    for (let r = 0, l = col.length; r < l; r += 1) {
+      const val = col[r];
+      let curCount = categoryCounts.get(val);
+      if (curCount === undefined) curCount = 0;
+      categoryCounts.set(val, curCount + 1);
+    }
+  }
+  return {
+    categorical: true,
+    categories: [...categoryCounts.keys()],
+    categoryCounts,
+    numCategories: categoryCounts.size
+  };
+}

client/src/util/dataframe/util.js

@@ -0,0 +1,28 @@
+/*
+Private utility code for dataframe
+*/
+
+export function isTypedArray(x) {
+  return (
+    ArrayBuffer.isView(x) &&
+    Object.prototype.toString.call(x) !== "[object DataView]"
+  );
+}
+
+export function isArrayOrTypedArray(x) {
+  return Array.isArray(x) || isTypedArray(x);
+}
+
+export function callOnceLazy(f) {
+  let value;
+  let calledOnce = false;
+  const result = function result(...args) {
+    if (!calledOnce) {
+      value = f(...args);
+      calledOnce = true;
+    }
+    return value;
+  };
+
+  return result;
+}

client/src/util/stateManager/controlsHelpers.js

@@ -0,0 +1,164 @@
+/*
+Helper functions for the controls reducer
+*/
+
+import _ from "lodash";
+
+import * as globals from "../../globals";
+import { fillRange } from "../typedCrossfilter/util";
+import {
+  userDefinedDimensionName,
+  diffexpDimensionName
+} from "../nameCreators";
+import * as World from "./world";
+
+/*
+Selection state for categoricals are tracked in an Object that
+has two main components for each category:
+1. mapping of option value to an index
+2. array of bool selection state by index
+Remember that option values can be ANY js type, except undefined/null.
+
+  {
+    _category_name_1: {
+      // map of option value to index
+      categoryIndices: Map([
+        catval1: index,
+        ...
+      ])
+
+      // index->selection true/false state
+      categorySelected: [ true/false, true/false, ... ]
+
+      // number of options
+      numCategories: number,
+
+      // isTruncated - true if the options for selection has
+      // been truncated (ie, was too large to implement)
+    }
+  }
+*/
+function topNCategories(summary) {
+  const counts = _.map(summary.categories, cat =>
+    summary.categoryCounts.get(cat)
+  );
+  const sortIndex = fillRange(new Array(summary.numCategories)).sort(
+    (a, b) => counts[b] - counts[a]
+  );
+  const sortedCategories = _.map(sortIndex, i => summary.categories[i]);
+  const sortedCounts = _.map(sortIndex, i => counts[i]);
+  const N = globals.maxCategoricalOptionsToDisplay;
+
+  if (sortedCategories.length < N) {
+    return [sortedCategories, sortedCounts];
+  }
+  return [sortedCategories.slice(0, N), sortedCounts.slice(0, N)];
+}
+
+export function createCategoricalSelectionState(state, world) {
+  const res = {};
+  _.forEach(world.obsAnnotations.colIndex.keys(), key => {
+    const summary = world.obsAnnotations.col(key).summarize();
+    if (summary.categories) {
+      const isColorField = key.includes("color") || key.includes("Color");
+      const isSelectableCategory =
+        !isColorField &&
+        key !== "name" &&
+        summary.categories.length < state.maxCategoryItems;
+      if (isSelectableCategory) {
+        const [categoryValues, categoryCounts] = topNCategories(summary);
+        const categoryIndices = new Map(categoryValues.map((v, i) => [v, i]));
+        const numCategories = categoryIndices.size;
+        const categorySelected = new Array(numCategories).fill(true);
+        const isTruncated = categoryValues.length < summary.numCategories;
+        res[key] = {
+          categoryValues, // array: of natively typed category values
+          categoryIndices, // map: category value (native type) -> category index
+          categorySelected, // array: t/f selection state
+          numCategories, // number: of categories
+          isTruncated, // bool: true if list was truncated
+          categoryCounts // array: cardinality of each category
+        };
+      }
+    }
+  });
+  return res;
+}
+
+/*
+given a categoricalSelectionState, return the list of all category values
+where selection state is true (ie, they are selected).
+*/
+export function selectedValuesForCategory(categorySelectionState) {
+  const selectedValues = _([...categorySelectionState.categoryIndices])
+    .filter(tuple => categorySelectionState.categorySelected[tuple[1]])
+    .map(tuple => tuple[0])
+    .value();
+  return selectedValues;
+}
+
+/*
+build a crossfilter dimension map for all gene expression related dimensions.
+*/
+export function createGenesDimMap(
+  userDefinedGenes,
+  diffexpGenes,
+  world,
+  crossfilter
+) {
+  function _createGenesDimMap(genes, nameCreator) {
+    return genes.reduce((acc, gene) => {
+      acc[nameCreator(gene)] = World.createVarDataDimension(
+        world,
+        crossfilter,
+        gene
+      );
+      return acc;
+    }, {});
+  }
+
+  return {
+    ..._createGenesDimMap(userDefinedGenes, userDefinedDimensionName),
+    ..._createGenesDimMap(diffexpGenes, diffexpDimensionName)
+  };
+}
+
+export function pruneVarDataCache(varData, needed) {
+  /*
+  Remove any unneeded columns from the varData dataframe.  Will only
+  prune / remove if the total column count exceeds VarDataCacheLowWatermark
+
+  Note: this code leverages the fact that dataframe offsets indicate
+  the order in which the columns were added.  This crudely provides
+  LRU semantics, so we can delete "older" columns first.
+  */
+
+  /*
+  VarDataCacheLowWatermark - this cofig value sets the minimum cache size,
+  in columns, below which we don't throw away data.
+
+  The value should be high enough so we are caching the maximum which will
+  "typically" be used in the UI (currently: 10 for diffexp, and N for user-
+  specified genes), and low enough to account for memory use (any single
+  column size is 4 bytes * numObs, so a column can be multi-megabyte in common
+  use cases).
+  */
+  const VarDataCacheLowWatermark = 32;
+
+  const numOverWatermark = varData.dims[1] - VarDataCacheLowWatermark;
+  if (numOverWatermark <= 0) return varData;
+
+  const { colIndex } = varData;
+  const all = colIndex.keys();
+  const unused = _.difference(all, needed);
+  if (unused.length > 0) {
+    // sort by offset in the dataframe - ie, psuedo-LRU
+    unused.sort((a, b) => colIndex.getOffset(a) - colIndex.getOffset(b));
+    const numToDrop =
+      unused.length < numOverWatermark ? unused.length : numOverWatermark;
+    for (let i = 0; i < numToDrop; i += 1) {
+      varData = varData.dropCol(unused[i]);
+    }
+  }
+  return varData;
+}

client/src/util/stateManager/index.js

@@ -16,5 +16,5 @@ exists to support those concepts.
 
 export * as Universe from "./universe";
 export * as World from "./world";
-export * as kvCache from "./keyvalcache";
 export * as WorldUtil from "./worldUtil";
+export * as ControlsHelper from "./controlsHelpers";

client/src/util/stateManager/keyvalcache.js

@@ -1,122 +0,0 @@
-// jshint esversion: 6
-import _ from "lodash";
-
-/*
-Very simple key/value cache for use by World & Universe.   Cache keys must
-be a string, and values are any JS non-primitive value.
-
-  * constructor(lowWatermark, minTTL):
-      - lowWatermark defines the number of cache elements below which
-        flushing will not occur.
-      - minTTL defines minimum time in milliseconds that cache entries will live.
-        A value of -1 disables automatic flushing (flush() can still
-        be called by external user).
-  * set() - add a key/val pair.
-  * get() - get a value or undefined if not present.
-  * flush(minAgeMs) - flush cache entries in excess of lowWatermark if those
-    entries are older than minAgeMs.
-
-*/
-
-const cachePrivateKey = "__kvcachekey__";
-const defaultLowWatermark = 32;
-const defaultMinTTL = 1000;
-
-function create(lowWatermark = defaultLowWatermark, minTTL = defaultMinTTL) {
-  if (typeof minTTL !== "number" || typeof lowWatermark !== "number") {
-    throw new TypeError(
-      "minTTL and lowWatermark parameters must be a primitive number"
-    );
-  }
-  if (lowWatermark < 0 || minTTL < 0) {
-    throw new RangeError(
-      "minTTL and lowWatermark parameters must be number greater than zero"
-    );
-  }
-
-  return {
-    [cachePrivateKey]: {
-      lowWatermark,
-      minTTL
-    }
-  };
-}
-
-function get(kvcache, key) {
-  if (key === cachePrivateKey) {
-    throw new RangeError(`key parameter may not have value ${cachePrivateKey}`);
-  }
-
-  const val = kvcache[key];
-  if (val) {
-    val[cachePrivateKey] = Date.now();
-  }
-  return val;
-}
-
-function set(kvcache, key, val) {
-  if (key === cachePrivateKey) {
-    throw new RangeError(`key parameter may not have value ${cachePrivateKey}`);
-  }
-
-  const newKvCache = { ...kvcache };
-  newKvCache[key] = val;
-  val[cachePrivateKey] = Date.now();
-  flushInPlace(newKvCache);
-  return newKvCache;
-}
-
-function flush(kvcache) {
-  const newKvCache = { ...kvcache };
-  flushInPlace(newKvCache);
-  return newKvCache;
-}
-
-/*
-Flush elements from cache IF cache size is greater than lowWatermark, and
-those elements are older than minAgeMS
-*/
-function flushInPlace(kvCache) {
-  const { lowWatermark, minTTL } = kvCache[cachePrivateKey];
-  const eol = Date.now() - minTTL;
-  const allKeys = _(kvCache)
-    .keys()
-    .filter(k => k !== cachePrivateKey)
-    .sortBy([k => kvCache[k][cachePrivateKey]])
-    .value();
-
-  if (allKeys.length > lowWatermark) {
-    const keysToDelete = _(allKeys)
-      .slice(0, allKeys.length - lowWatermark)
-      .filter(k => kvCache[k][cachePrivateKey] <= eol)
-      .value();
-    _.forEach(keysToDelete, k => delete kvCache[k]);
-  }
-
-  return kvCache;
-}
-
-/*
-use to create a cache that is a transformation of another cache.
-*/
-function map(srcKvCache, cb, createOptions) {
-  const keysInSrcKvCache = _(srcKvCache)
-    .keys()
-    .filter(k => k !== cachePrivateKey)
-    .value();
-  const lowWatermark = _.get(
-    createOptions,
-    "lowWatermark",
-    defaultLowWatermark
-  );
-  const minTTL = _.get(createOptions, "minTTL", defaultMinTTL);
-  const newKvCache = create(lowWatermark, minTTL);
-  _.forEach(keysInSrcKvCache, key => {
-    const val = cb(get(srcKvCache, key), key);
-    newKvCache[key] = val;
-    val[cachePrivateKey] = Date.now();
-  });
-  return newKvCache;
-}
-
-export { create, get, set, flush, map };

client/src/util/stateManager/summarizeAnnotations.js

@@ -1,122 +0,0 @@
-import _ from "lodash";
-import finiteExtent from "../finiteExtent";
-
-/*
-Build and return obs/var summary using any annotation in the schema
-
-Summary information for each annotation, keyed by annotation name.
-Value will be an object, containing summary information.
-
-For continuous annotations (int, float, etc):
-  <annotation_name>: {
-    categorical: false,
-    range {
-      min: <number>,
-      max: <number>
-    }
-  }
-
-For categorical annotations (boolean, string, category):
-  <annotation_name>: {
-    categorical: true,
-    categories: [ <category1>, <category2>, ... ]
-    categoryCounts: Map {
-      <category1>: <number>,
-      ...
-    },
-    numCategories: <number>
-  }
-
-Summarize will be returned for BOTH obs and var annotations.
-
-Example:
-  {
-    "Splice_sites_Annotated": {
-      categorical: false,
-      range: {
-        "min": 26,
-        "max": 1075869
-      }
-    },
-    "Selection": {
-      categorical: true,
-      numCategories, 3,
-      categories: [ "Astrocytes(HEPACAM)", "Endothelial(BSC)", "Unpanned" ],
-      categoryCounts: Map {
-        "Astrocytes(HEPACAM)": 714,
-        "Endothelial(BSC)": 123,
-        "Unpanned": 665
-      }
-    }
-  }
-
-NOTE: will not summarize the required 'name' annotation, as that is
-specified as unique per element.
-*/
-function _summarizeAnnotations(_schema, annotations) {
-  const summary = _(_schema) // lodash wrapping: https://lodash.com/docs/4.17.11#lodash
-    .filter(v => v.name !== "name")
-    .keyBy("name")
-    .mapValues(anno => {
-      const { name, type } = anno;
-      const continuous = type === "int32" || type === "float32";
-
-      if (continuous) {
-        let min;
-        let max;
-        let nan = 0;
-        let pinf = 0;
-        let ninf = 0;
-        for (let r = 0; r < annotations.length; r += 1) {
-          const val = Number(annotations[r][name]);
-          if (Number.isFinite(val)) {
-            if (min === undefined) {
-              min = val;
-              max = val;
-            } else {
-              min = val < min ? val : min;
-              max = val > max ? val : max;
-            }
-          } else if (Number.isNaN(val)) {
-            nan += 1;
-          } else if (val > 0) {
-            pinf += 1;
-          } else {
-            ninf += 1;
-          }
-        }
-        return {
-          categorical: false,
-          range: { min, max, nan, pinf, ninf }
-        };
-      }
-
-      /* else categorical */
-      const categoryCounts = new Map();
-      for (let r = 0; r < annotations.length; r += 1) {
-        const val = annotations[r][name];
-        let curCount = categoryCounts.get(val);
-        if (curCount === undefined) curCount = 0;
-        categoryCounts.set(val, curCount + 1);
-      }
-      return {
-        categorical: true,
-        categories: [...categoryCounts.keys()],
-        categoryCounts,
-        numCategories: categoryCounts.size
-      };
-    })
-    .value();
-  return summary;
-}
-
-export default function summarizeAnnotations(
-  schema,
-  obsAnnotations,
-  varAnnotations
-) {
-  return {
-    obs: _summarizeAnnotations(schema.annotations.obs, obsAnnotations),
-    var: _summarizeAnnotations(schema.annotations.var, varAnnotations)
-  };
-}

client/src/util/stateManager/universe.js

@@ -2,24 +2,15 @@
 
 import _ from "lodash";
 
-import * as kvCache from "./keyvalcache";
-import summarizeAnnotations from "./summarizeAnnotations";
 import decodeMatrixFBS from "./matrix";
+import * as Dataframe from "../dataframe";
 
 /*
 Private helper function - create and return a template Universe
 */
 function templateUniverse() {
   /* default universe template */
-
-  /* varDataCache config - see kvCache for semantics */
-  const VarDataCacheLowWatermark = 32; // cache element count
-  const VarDataCacheTTLMs = 1000; // min cache time in MS
-
   return {
-    api: null,
-    finalized: false, // XXX: may not be needed
-
     nObs: 0,
     nVar: 0,
     schema: {},
@@ -27,21 +18,14 @@ function templateUniverse() {
     /*
     Annotations
     */
-    obsAnnotations: [] /* all obs annotations, by obs index */,
-    varAnnotations: [] /* all var annotations, by var index */,
-    obsNameToIndexMap: {} /* reverse map 'name' to index */,
-    varNameToIndexMap: {} /* reverse map 'name' to index */,
-    summary: null /* derived data summaries XXX: consider exploding in place */,
-
-    obsLayout: { X: [], Y: [] } /* xy layout */,
+    obsAnnotations: Dataframe.Dataframe.empty(),
+    varAnnotations: Dataframe.Dataframe.empty(),
+    obsLayout: Dataframe.Dataframe.empty(),
 
     /*
-    Cache of var data (expression), by var annotation name.   Data can be
-    accesses as a POJO, but if you want caching semantics, use the kvCache
-    API (eg., kvCache.get(), kvCache.set(), ...), which will maintain the
-    LRU semantics.
+    Var data columns - subset of all
     */
-    varDataCache: kvCache.create(VarDataCacheLowWatermark, VarDataCacheTTLMs)
+    varData: Dataframe.Dataframe.empty(null, new Dataframe.KeyIndex())
   };
 }
 
@@ -53,81 +37,29 @@ These functions are used exclusively by the actions and reducers to
 build an internal POJO for use by the rendering components.
 */
 
-/*
-generate any client-side transformations or summarization that
-is independent of REST API response formats.
-*/
-function finalize(universe) {
-  /* A bit of sanity checking! */
-  const { nObs, nVar } = universe;
-  if (
-    nObs !== universe.obsAnnotations.length ||
-    nObs !== universe.obsLayout.X.length ||
-    nObs !== universe.obsLayout.Y.length ||
-    nVar !== universe.varAnnotations.length
-  ) {
-    throw new Error("Universe dimensionality mismatch - failed to load");
-  }
-  // TODO: add more sanity checks, such as:
-  //  - all annotations in the schema
-  //  - layout has supported number of dimensions
-  //  - ...
-
+function AnnotationsFBSToDataframe(arrayBuffer) {
   /*
-  Create all derived (convenience) data structures.
-  */
-  universe.obsNameToIndexMap = _.transform(
-    universe.obsAnnotations,
-    (acc, value, idx) => {
-      acc[value.name] = idx;
-    },
-    {}
-  );
-  universe.varNameToIndexMap = _.transform(
-    universe.varAnnotations,
-    (acc, value, idx) => {
-      acc[value.name] = idx;
-    },
-    {}
-  );
-  universe.finalized = true;
-  return universe;
-}
-
-function RESTv02AnotationsFBSResponseToInternal(arrayBuffer) {
-  /*
-  Convert a Matrix FBS to our internal format -- row-major array of
-  observations/cells, stored as an object.  Each obs has a key for each
-  annotation, plus __index__ containing its obsIndex.
-
-  Example:
-  [
-    { __index__: 0, tissue_type: "lung", sex: "F", ... },
-    ...
-  ]
-
-  XXX TODO: we could make use of the columns in building crossfilter
-  dimensions (they have to be recreated).  Future optimization.
+  Convert a Matrix FBS to a Dataframe.
   */
   const fbs = decodeMatrixFBS(arrayBuffer);
-  const keys = fbs.colIdx;
-  const result = Array(fbs.nRows);
-  for (let row = 0; row < fbs.nRows; row += 1) {
-    const rec = { __index__: row };
-    for (let col = 0; col < fbs.nCols; col += 1) {
-      rec[keys[col]] = fbs.columns[col][row];
-    }
-    result[row] = rec;
-  }
-  return result;
+  const df = new Dataframe.Dataframe(
+    [fbs.nRows, fbs.nCols],
+    fbs.columns,
+    null,
+    new Dataframe.KeyIndex(fbs.colIdx)
+  );
+  return df;
 }
 
-function RESTv02LayoutFBSResponseToInternal(arrayBuffer) {
+function LayoutFBSToDataframe(arrayBuffer) {
   const fbs = decodeMatrixFBS(arrayBuffer, true);
-  return {
-    X: fbs.columns[0],
-    Y: fbs.columns[1]
-  };
+  const df = new Dataframe.Dataframe(
+    [fbs.nRows, fbs.nCols],
+    fbs.columns,
+    null,
+    new Dataframe.KeyIndex(["X", "Y"])
+  );
+  return df;
 }
 
 function reconcileSchemaCategoriesWithSummary(universe) {
@@ -149,14 +81,14 @@ function reconcileSchemaCategoriesWithSummary(universe) {
     ) {
       const categories = _.union(
         _.get(s, "categories", []),
-        _.get(universe.summary.obs[s.name], "categories", [])
+        _.get(universe.obsAnnotations.col(s.name).summarize(), "categories", [])
       );
       s.categories = categories;
     }
   });
 }
 
-export function createUniverseFromRestV02Response(
+export function createUniverseFromResponse(
   configResponse,
   schemaResponse,
   annotationsObsResponse,
@@ -169,33 +101,28 @@ export function createUniverseFromRestV02Response(
   const { schema } = schemaResponse;
   const universe = templateUniverse();
 
-  /* constants */
-  universe.api = "0.2";
-
   /* schema related */
   universe.schema = schema;
   universe.nObs = schema.dataframe.nObs;
   universe.nVar = schema.dataframe.nVar;
 
   /* annotations */
-  universe.obsAnnotations = RESTv02AnotationsFBSResponseToInternal(
-    annotationsObsResponse
-  );
-  universe.varAnnotations = RESTv02AnotationsFBSResponseToInternal(
-    annotationsVarResponse
-  );
-
+  universe.obsAnnotations = AnnotationsFBSToDataframe(annotationsObsResponse);
+  universe.varAnnotations = AnnotationsFBSToDataframe(annotationsVarResponse);
   /* layout */
-  universe.obsLayout = RESTv02LayoutFBSResponseToInternal(layoutFBSResponse);
+  universe.obsLayout = LayoutFBSToDataframe(layoutFBSResponse);
 
-  universe.summary = summarizeAnnotations(
-    universe.schema,
-    universe.obsAnnotations,
-    universe.varAnnotations
-  );
+  /* sanity check */
+  if (
+    universe.nObs !== universe.obsLayout.length ||
+    universe.nObs !== universe.obsAnnotations.length ||
+    universe.nVar !== universe.varAnnotations.length
+  ) {
+    throw new Error("Universe dimensionality mismatch - failed to load");
+  }
 
   reconcileSchemaCategoriesWithSummary(universe);
-  return finalize(universe);
+  return universe;
 }
 
 export function convertDataFBStoObject(universe, arrayBuffer) {
@@ -214,8 +141,8 @@ export function convertDataFBStoObject(universe, arrayBuffer) {
   const result = {};
 
   for (let c = 0; c < colIdx.length; c += 1) {
-    const gene = universe.varAnnotations[colIdx[c]].name;
-    result[gene] = columns[c];
+    const varName = universe.varAnnotations.at(colIdx[c], "name");
+    result[varName] = columns[c];
   }
   return result;
 }

client/src/util/stateManager/world.js

@@ -1,13 +1,12 @@
 // jshint esversion: 6
 
 import _ from "lodash";
-import * as kvCache from "./keyvalcache";
-import summarizeAnnotations from "./summarizeAnnotations";
 import { layoutDimensionName, obsAnnoDimensionName } from "../nameCreators";
 import Crossfilter from "../typedCrossfilter";
-import { sliceByIndex } from "../typedCrossfilter/util";
+import * as Dataframe from "../dataframe";
 
 /*
+
 World is a subset of universe.   Most code should use world, and should
 (generally) not use Universe.   World contains any per-obs or per-var data
 that must be consistent acorss the app when we view/manipulate subsets
@@ -16,120 +15,76 @@ of Universe.
 Private API indicated by leading underscore in key name (eg, _foo).  Anything else
 is public.
 
-World contains several public keys, obsAnnotations, and obsLayout, which are
-arrays contianing information about an OBS in the same order/offset.  In
-other words, world.obsAnnotations[0] and world.obsLayout.X[0] refer to the same
-obs/cell.
+Notable keys in the world object:
+
+* nObs, nVar: dimensions
+
+* schema: data schema from the server
 
 * obsAnnotations:
 
-  obsAnnotations will return an array of objects.  Each object contains all annotation
-  values for a given observation/cell, keyed by annotation name, PLUS a key
-  '__cellId__', containing a REST API ID for this obs/cell (referred to as the
-  obsIndex in the REST 0.2 spec or cellIndex in the 0.1 spec.
+  Dataframe containing obs annotations.  Columns are indexed by annotation
+  name (eg, 'tissue type'), and rows are indexed by the REST API obsIndex
+  (ie, the offset into the underlying server-side dataframe).
 
-  Example:  [ { __cellId__: 99, cluster: 'blue', numReads: 93933 } ]
-
-  NOTE: world.obsAnnotation should be identical to the old state.cells value,
-  EXCEPT that
-    * __cellIndex__ renamed to __index__
-    * __x__ and __y__ are now in world.obsLayout
-    * __color__ and __colorRBG__ should be moved to controls reducer
+  This indexing means that you can access data by _either_ the server's
+  obxIndex, or the offset into the client-side column array .  Be careful
+  to know which you want and are using.
 
 * obsLayout:
 
-  obsLayout will return an object containing two arrays, containing X and Y
-  coordinates respectively.
+  A dataframe containing the X/Y layout for all obs.  Columns are named
+  'X' and 'Y', and rows are indexed in the same way as obsAnnotation.
 
-  Example: { X: [ 0.33, 0.23, ... ], Y: [ 0.8, 0.777, ... ]}
-
-* crossfilter - a crossfilter object across world.obsAnnotations
-
-* dimensionMap - an object mapping annotation names to dimensions on
-  the crossfilter
+* varData: a cache of expression columns, stored in a Dataframe.  Cache
+  managed by controls reducer.
 
 */
 
-/* varDataCache config - see kvCache for semantics */
-const VarDataCacheLowWatermark = 32; // cache element count
-const VarDataCacheTTLMs = 1000; // min cache time in MS
-
 function templateWorld() {
   return {
-    // map from universe obsIndex to world offset.
-    // Undefined / null indicates identity mapping.
-    obsIndex: null,
-    obsBackIndex: null,
-
     /* schema/version related */
-    api: null,
     schema: null,
     nObs: 0,
     nVar: 0,
 
     /* annotations */
-    obsAnnotations: null,
-    varAnnotations: null,
+    obsAnnotations: Dataframe.Dataframe.empty(),
+    varAnnotations: Dataframe.Dataframe.empty(),
 
-    /* layout of graph */
-    obsLayout: null,
+    /* layout of graph. Dataframe. */
+    obsLayout: Dataframe.Dataframe.empty(),
 
-    /* derived data summaries XXX: consider exploding in place */
-    summary: null,
-
-    varDataCache: kvCache.create(
-      VarDataCacheLowWatermark,
-      VarDataCacheTTLMs
-    ) /* cache of var data (expression) */
+    /*
+    Var data columns - subset of all data (may be empty)
+    */
+    varData: Dataframe.Dataframe.empty(null, new Dataframe.KeyIndex())
   };
 }
 
 export function createWorldFromEntireUniverse(universe) {
-  if (!universe.finalized) {
-    throw new Error("World can't be created from an partial Universe");
-  }
-
   const world = templateWorld();
 
-  // map from the universe obsIndex to our world offset.
-  // undefined/null indicates identity map.
-  // In other words   obsBackIndex[universeIdx] -> worldIdx
-  world.obsBackIndex = null;
-  // Map to the universe index for each element in world.
-  // Null indicates identity map (aka world === universe)
-  // In other wrods  obsIndex[worldIdx]  ->  universeIdx
-  world.obsIndex = null;
-
   /*
   public interface follows
   */
 
   /* Schema related */
-  world.api = universe.api;
   world.schema = universe.schema;
   world.nObs = universe.nObs;
   world.nVar = universe.nVar;
 
-  /* annotations */
+  /* annotation dataframes */
   world.obsAnnotations = universe.obsAnnotations;
   world.varAnnotations = universe.varAnnotations;
 
-  /* layout and display characteristics */
+  /* layout and display characteristics dataframe */
   world.obsLayout = universe.obsLayout;
 
-  /* derived data & summaries */
-  world.summary = summarizeAnnotations(
-    world.schema,
-    world.obsAnnotations,
-    world.varAnnotations
-  );
-
-  /* build the varDataCache */
-  world.varDataCache = kvCache.map(
-    universe.varDataCache,
-    val => subsetVarData(world, universe, val),
-    { lowWatermark: VarDataCacheLowWatermark, minTTL: VarDataCacheTTLMs }
-  );
+  /*
+  Var data columns - subset of all
+  */
+  world.varData = universe.varData.clone();
 
   return world;
 }
@@ -138,54 +93,24 @@ export function createWorldFromCurrentSelection(universe, world, crossfilter) {
   const newWorld = templateWorld();
 
   /* these don't change as only OBS are selected in our current implementation */
-  newWorld.api = universe.api;
   newWorld.nVar = universe.nVar;
   newWorld.schema = universe.schema;
   newWorld.varAnnotations = universe.varAnnotations;
 
-  /* build index maps and back maps based upon current selection state */
-  const obsBackIndex = new Uint32Array(universe.nObs);
-  obsBackIndex.fill(-1); // default - aka unused
-  const notSelected = obsBackIndex[0];
-  let nObs = 0;
-  for (let i = 0; i < universe.nObs; i += 1) {
-    if (crossfilter.isElementFiltered(i)) {
-      obsBackIndex[i] = nObs;
-      nObs += 1;
-    }
+  /* now subset/cut obs */
+  const mask = crossfilter.allFilteredMask();
+  newWorld.obsAnnotations = world.obsAnnotations.isubsetMask(mask);
+  newWorld.obsLayout = world.obsLayout.isubsetMask(mask);
+  newWorld.nObs = newWorld.obsAnnotations.dims[0];
+
+  /*
+  Var data columns - subset of all
+  */
+  if (world.varData.isEmpty()) {
+    newWorld.varData = world.varData.clone();
+  } else {
+    newWorld.varData = world.varData.isubsetMask(mask);
   }
-  const obsIndex = new Uint32Array(nObs);
-  for (let i = 0; i < universe.nObs; i += 1) {
-    const worldIdx = obsBackIndex[i];
-    if (worldIdx !== notSelected) {
-      obsIndex[worldIdx] = i;
-    }
-  }
-
-  newWorld.nObs = nObs;
-  newWorld.obsIndex = obsIndex;
-  newWorld.obsBackIndex = obsBackIndex;
-
-  /* now slice */
-  newWorld.obsAnnotations = sliceByIndex(universe.obsAnnotations, obsIndex);
-  newWorld.obsLayout = {
-    X: sliceByIndex(universe.obsLayout.X, obsIndex),
-    Y: sliceByIndex(universe.obsLayout.Y, obsIndex)
-  };
-
-  /* derived data & summaries */
-  newWorld.summary = summarizeAnnotations(
-    newWorld.schema,
-    newWorld.obsAnnotations,
-    newWorld.varAnnotations
-  );
-
-  /* build the varDataCache */
-  newWorld.varDataCache = kvCache.map(
-    universe.varDataCache,
-    val => subsetVarData(newWorld, universe, val),
-    { lowWatermark: VarDataCacheLowWatermark, minTTL: VarDataCacheTTLMs }
-  );
   return newWorld;
 }
 
@@ -225,17 +150,10 @@ function deduceDimensionType(attributes, fieldName) {
   when it is no longer needed
   (it will not be garbage collected without this call)
 */
-
-export function createVarDimension(
-  world,
-  _worldVarDataCache,
-  crossfilter,
-  geneName
-) {
-  // return crossfilter.dimension(_worldVarDataCache[geneName], Float32Array);
+export function createVarDataDimension(world, crossfilter, name) {
   return crossfilter.dimension(
     Crossfilter.ScalarDimension,
-    _worldVarDataCache[geneName],
+    world.varData.col(name).asArray(),
     Float32Array
   );
 }
@@ -245,22 +163,23 @@ export function createObsDimensionMap(crossfilter, world) {
   create and return a crossfilter dimension for every obs annotation
   for which we have a supported type.
   */
-  const { schema, obsLayout } = world;
+  const { schema, obsLayout, obsAnnotations } = world;
 
   // Create a crossfilter dimension for all obs annotations *except* 'name'
   const dimensionMap = _(schema.annotations.obs)
     .filter(anno => anno.name !== "name")
     .transform((result, anno) => {
       const dimType = deduceDimensionType(anno, anno.name);
+      const colData = obsAnnotations.col(anno.name).asArray();
       if (dimType === "enum") {
         result[obsAnnoDimensionName(anno.name)] = crossfilter.dimension(
           Crossfilter.EnumDimension,
-          r => r[anno.name]
+          colData
         );
-      } else {
+      } else if (dimType) {
         result[obsAnnoDimensionName(anno.name)] = crossfilter.dimension(
           Crossfilter.ScalarDimension,
-          r => r[anno.name],
+          colData,
           dimType
         );
       } // else ignore the annotation
@@ -272,8 +191,8 @@ export function createObsDimensionMap(crossfilter, world) {
   */
   dimensionMap[layoutDimensionName("XY")] = crossfilter.dimension(
     Crossfilter.SpatialDimension,
-    obsLayout.X,
-    obsLayout.Y
+    obsLayout.col("X").asArray(),
+    obsLayout.col("Y").asArray()
   );
 
   return dimensionMap;
@@ -283,10 +202,20 @@ export function worldEqUniverse(world, universe) {
   return world.obsAnnotations === universe.obsAnnotations;
 }
 
-export function subsetVarData(world, universe, varData) {
-  // If world === universe, just return the entire varData array
-  if (worldEqUniverse(world, universe)) {
-    return varData;
+export function getSelectedByIndex(crossfilter) {
+  /*
+  return array of obsIndex, containing all selected obs/cells.
+  */
+  const selected = crossfilter.allFilteredMask(); // array of bool-ish
+  const keys = crossfilter.data.rowIndex.keys(); // row keys, aka universe rowIndex
+
+  const set = new Int32Array(selected.length);
+  let numElems = 0;
+  for (let i = 0, l = selected.length; i < l; i += 1) {
+    if (selected[i]) {
+      set[numElems] = keys[i];
+      numElems += 1;
+    }
   }
-  return sliceByIndex(varData, world.obsIndex);
+  return new Int32Array(set.buffer, 0, numElems);
 }

client/src/util/stateManager/worldUtil.js

@@ -18,13 +18,23 @@ Map {
    ...
 }
 
+Parameters are:
+- dim1: dimension 1 name/label
+- dim2: dimension 2 name/label
+- df: dataframe containing dim1 and dim2 on the column axis
+
 */
-function _countCategoryValues2D(dim1, dim2, rows) {
+function _countCategoryValues2D(dim1, dim2, df) {
   const dimMap = new Map();
-  for (let r = 0; r < rows.length; r += 1) {
-    const row = rows[r];
-    const val1 = row[dim1];
-    const val2 = row[dim2];
+  const col1 = df.col(dim1) ? df.col(dim1).asArray() : null;
+  const col2 = df.col(dim2) ? df.col(dim2).asArray() : null;
+  if (!col1 || !col2) {
+    return dimMap;
+  }
+
+  for (let r = 0, l = df.length; r < l; r += 1) {
+    const val1 = col1[r];
+    const val2 = col2[r];
     let d2Map = dimMap.get(val1);
     if (d2Map === undefined) {
       d2Map = new Map();

client/src/util/typedCrossfilter/bitArray.js

@@ -40,7 +40,7 @@ class BitArray {
   // Return the number of records that are selected, ie, have a one bit in
   // all allocated dimensions.
   //
-  get selectionCount() {
+  selectionCount() {
     return this.countAllOnes();
   }
 
@@ -48,16 +48,27 @@ class BitArray {
   //
   countAllOnes() {
     let count = 0;
-    const { bitarray, bitmask, length, width } = this;
-    for (let l = 0; l < length; l += 1) {
-      let dimensionsSet = 0;
-      for (let w = 0; w < width; w += 1) {
-        if (bitarray[w * length + l] === bitmask[w]) {
-          dimensionsSet += 1;
+    const { bitarray, length, width } = this;
+    if (width === 1) {
+      // special case, width === 1, for performance
+      const bitmask = this.bitmask[0];
+      for (let l = 0; l < length; l += 1) {
+        if (bitarray[l] === bitmask) {
+          count += 1;
         }
       }
-      if (dimensionsSet === width) {
-        count += 1;
+    } else {
+      const { bitmask } = this;
+      for (let l = 0; l < length; l += 1) {
+        let dimensionsSet = 0;
+        for (let w = 0; w < width; w += 1) {
+          if (bitarray[w * length + l] === bitmask[w]) {
+            dimensionsSet += 1;
+          }
+        }
+        if (dimensionsSet === width) {
+          count += 1;
+        }
       }
     }
     return count;
@@ -233,12 +244,14 @@ class BitArray {
   fillBySelection(result, selectedValue, deselectedValue) {
     // special case (width === 1) for performance
     if (this.width === 1) {
-      const bitmask = this.bitmask[0];
-      for (let i = 0, len = this.length; i < len; i += 1) {
-        result[i] =
-          bitmask && this.bitarray[i] === bitmask
-            ? selectedValue
-            : deselectedValue;
+      const { bitmask, bitarray } = this;
+      const mask = bitmask[0];
+      if (!mask) {
+        result.fill(deselectedValue);
+      } else {
+        for (let i = 0, len = this.length; i < len; i += 1) {
+          result[i] = bitarray[i] === mask ? selectedValue : deselectedValue;
+        }
       }
     } else {
       for (let i = 0, len = this.length; i < len; i += 1) {

client/src/util/typedCrossfilter/index.js

@@ -39,6 +39,14 @@ import {
   upperBoundIndirect
 } from "./util";
 
+function isArrayOrTypedArray(x) {
+  return (
+    Array.isArray(x) ||
+    (ArrayBuffer.isView(x) &&
+      Object.prototype.toString.call(x) !== "[object DataView]")
+  );
+}
+
 class NotImplementedError extends Error {
   constructor(...params) {
     super(...params);
@@ -52,6 +60,11 @@ class NotImplementedError extends Error {
 
 class TypedCrossfilter {
   constructor(data) {
+    /*
+    Typically, data is one of:
+      - Array of objects/records
+      - Dataframe (util/dataframe)
+    */
     this.data = data;
 
     // filters: array of { id, dimension }
@@ -97,18 +110,32 @@ class TypedCrossfilter {
   // return array of all records that are selected/filtered
   // by all dimensions.
   allFiltered() {
-    const { selection } = this;
-    const res = [];
-    for (let i = 0, len = this.data.length; i < len; i += 1) {
-      if (selection.isSelected(i)) {
-        res.push(this.data[i]);
+    const { data, selection } = this;
+    if (Array.isArray(data)) {
+      const res = [];
+      for (let i = 0, len = data.length; i < len; i += 1) {
+        if (selection.isSelected(i)) {
+          res.push(data[i]);
+        }
       }
+      return res;
     }
-    return res;
+    /* else, Dataframe-like */
+    return data.isubsetMask(this.allFilteredMask());
+  }
+
+  // return Uint8array containing selection state (truthy/falsey) for each record.
+  //
+  allFilteredMask() {
+    return this.selection.fillBySelection(
+      new Uint8Array(this.data.length),
+      1,
+      0
+    );
   }
 
   countFiltered() {
-    return this.selection.selectionCount;
+    return this.selection.selectionCount();
   }
 
   isElementFiltered(i) {
@@ -161,6 +188,7 @@ class ScalarDimension extends _Dimension {
     // or a map function which will create it.
     let array;
     if (value instanceof ValueArrayType) {
+      // user has provided the final typed array - just use it
       if (value.length !== this.crossfilter.data.length) {
         throw new RangeError(
           "ScalarDimension values length must equal crossfilter data record count"
@@ -168,11 +196,18 @@ class ScalarDimension extends _Dimension {
       }
       array = value;
     } else if (value instanceof Function) {
-      // Create value array
+      // Create value array from user-provided map function.
       array = this._createValueArray(
         value,
         new ValueArrayType(this.crossfilter.data.length)
       );
+    } else if (isArrayOrTypedArray(value)) {
+      // Create value array from user-provided array.  Typically used
+      // only by enumerated dimensions
+      array = this._createValueArray(
+        i => value[i],
+        new ValueArrayType(this.crossfilter.data.length)
+      );
     } else {
       throw new NotImplementedError(
         "dimension value must be function or value array type"
@@ -190,7 +225,7 @@ class ScalarDimension extends _Dimension {
     const len = data.length;
     const larray = array;
     for (let i = 0; i < len; i += 1) {
-      larray[i] = value(data[i]);
+      larray[i] = value(i, data);
     }
     return larray;
   }
@@ -387,7 +422,7 @@ class EnumDimension extends ScalarDimension {
     // and the enum.
     const s = new Set();
     for (let i = 0; i < len; i += 1) {
-      s.add(value(data[i]));
+      s.add(value(i, data));
     }
     this.enumIndex = Array.from(s);
     this.enumIndex.sort();
@@ -395,7 +430,7 @@ class EnumDimension extends ScalarDimension {
     // create dimension value array
     const enumLen = this.enumIndex.length;
     for (let i = 0; i < len; i += 1) {
-      const v = value(data[i]);
+      const v = value(i, data);
       const e = lowerBound(this.enumIndex, v, 0, enumLen);
       larray[i] = e;
     }

makefile

@@ -15,6 +15,7 @@ build-server : build-client
 	cp -r server/* $(SERVERBUILD)
 	cp -r client/build/  $(CLIENTBUILD)
 	mkdir -p $(SERVERBUILD)/app/web/static/img
+	mkdir -p $(SERVERBUILD)/app/web/templates/
 	cp $(CLIENTBUILD)/index.html $(SERVERBUILD)/app/web/templates/
 	cp -r $(CLIENTBUILD)/static $(SERVERBUILD)/app/web/
 	cp $(CLIENTBUILD)/favicon.png $(SERVERBUILD)/app/web/static/img
@@ -28,6 +29,8 @@ build-client :
 # If you are actively developing in the server folder use this, dirties the source tree
 build-for-server-dev : clean-server build-client
 	mkdir -p server/app/web/static/img
+	mkdir -p server/app/web/static/js
+	mkdir -p server/app/web/templates/
 	cp client/build/index.html server/app/web/templates/
 	cp -r client/build/static server/app/web/
 	cp client/build/favicon.png server/app/web/static/img

server/app/app.py

@@ -4,7 +4,6 @@ from flask import Flask
 from flask_caching import Cache
 from flask_compress import Compress
 from flask_cors import CORS
-from flask_restful_swagger_2 import get_swagger_blueprint
 
 from .rest_api.rest import get_api_resources
 from .util.utils import Float32JSONEncoder
@@ -26,21 +25,7 @@ app.config.update(SECRET_KEY=SECRET_KEY)
 # Application Data
 data = None
 
-# A list of swagger document objects
-docs = []
 resources = get_api_resources()
-docs.append(resources.get_swagger_doc())
-
 app.register_blueprint(webapp.bp)
 app.register_blueprint(resources.blueprint)
-app.register_blueprint(
-    get_swagger_blueprint(
-        docs,
-        "/api/swagger",
-        produces=["application/json"],
-        title="cellxgene rest api",
-        description="An API connecting ExpressionMatrix2 clustering algorithm to cellxgene",
-    )
-)
-
 app.add_url_rule("/", endpoint="index")

server/app/driver/driver.py

@@ -42,45 +42,12 @@ class CXGDriver(metaclass=ABCMeta):
         pass
 
     @abstractmethod
-    def filter_dataframe(self, filter):
-        """
-        Filter cells from data and return a subset of the data. They can operate on both obs and var dimension with
-        indexing and filtering by annotation value. Filters are combined with the and operator.
-        See REST specs for info on filter format:
-        https://github.com/chanzuckerberg/cellxgene/blob/master/docs/REST_API.md
-
-        :param filter: dictionary with filter params
-        :return: View into scanpy object with cells/genes filtered
-        """
-        pass
-
-    @abstractmethod
-    def annotation(self, filter, axis, fields=None):
+    def annotation_to_fbs_matrix(self, axis, field=None):
         """
         Gets annotation value for each observation
-        :param filter: filter: dictionary with filter params
         :param axis: string obs or var
         :param fields: list of keys for annotation to return, returns all annotation values if not set.
-        :return: dict: names - list of fields in order, data - list of lists or metadata
-        [observation ids, val1, val2...]
-        """
-        pass
-
-    @abstractmethod
-    def annotation_to_fbs_matrix(self, axis, field=None):
-        """ Same as annotation(), except returns a flatbuffer, and does not support filtering. """
-        pass
-
-    @abstractmethod
-    def data_frame(self, filter, axis):
-        """
-        Retrieves data for each variable for observations in data frame
-        :param filter: filter: dictionary with filter params
-        :param axis: string obs or var
-        :return: {
-            "var": list of variable ids,
-            "obs": [cellid, var1 expression, var2 expression, ...],
-        }
+        :return: flatbuffer: in fbs/matrix.fbs encoding
         """
         pass
 
@@ -104,16 +71,6 @@ class CXGDriver(metaclass=ABCMeta):
         """
         pass
 
-    @abstractmethod
-    def layout(self, filter, interactive_limit=None):
-        """
-        Computes a n-d layout for cells through dimensionality reduction.
-        :param filter: filter: dictionary with filter params
-        :param interactive_limit: -- don't compute if total # genes in dataframes are larger than this
-        :return:  [cellid, x, y, ...]
-        """
-        pass
-
     @abstractmethod
     def layout_to_fbs_matrix(self, filter):
         """ same as layout, except returns a flatbuffer """

server/app/rest_api/rest.py

@@ -3,22 +3,17 @@ import pkg_resources
 import warnings
 
 from flask import Blueprint, current_app, jsonify, make_response, request
-from flask_restful_swagger_2 import Api, swagger, Resource
-from werkzeug.datastructures import ImmutableMultiDict
+from flask_restful import Api, Resource
 
 from server.app.util.constants import (
     Axis,
     DiffExpMode,
     JSON_NaN_to_num_warning_msg,
 )
-from server.app.util.filter import parse_filter, QueryStringError
-from server.app.util.models import FilterModel
-from server.app.util.utils import get_mime_type
 from server.app.util.errors import (
     FilterError,
     InteractiveError,
     JSONEncodingValueError,
-    MimeTypeError,
     PrepareError,
 )
 
@@ -31,47 +26,6 @@ Sort order for routes
 
 
 class SchemaAPI(Resource):
-    @swagger.doc(
-        {
-            "summary": "get schema for dataframe and annotations",
-            "tags": ["initialize"],
-            "parameters": [],
-            "responses": {
-                "200": {
-                    "description": "schema",
-                    "examples": {
-                        "application/json": {
-                            "schema": {
-                                "dataframe": {
-                                    "nObs": 383,
-                                    "nVar": 19944,
-                                    "type": "float32",
-                                },
-                                "annotations": {
-                                    "obs": [
-                                        {"name": "name", "type": "string"},
-                                        {"name": "tissue_type", "type": "string"},
-                                        {"name": "num_reads", "type": "int32"},
-                                        {"name": "sample_name", "type": "string"},
-                                        {
-                                            "name": "clusters",
-                                            "type": "categorical",
-                                            "categories": [99, 1, "unknown cluster"],
-                                        },
-                                        {"name": "QScore", "type": "float32"},
-                                    ],
-                                    "var": [
-                                        {"name": "name", "type": "string"},
-                                        {"name": "gene", "type": "string"},
-                                    ],
-                                },
-                            }
-                        }
-                    },
-                }
-            },
-        }
-    )
     def get(self):
         return make_response(
             jsonify({"schema": current_app.data.schema}), HTTPStatus.OK
@@ -79,47 +33,6 @@ class SchemaAPI(Resource):
 
 
 class ConfigAPI(Resource):
-    @swagger.doc(
-        {
-            "summary": "Configuration information to assist in front-end adaptation"
-            " to underlying engine, available functionality, interactive time limits, etc",
-            "tags": ["initialize"],
-            "parameters": [],
-            "responses": {
-                "200": {
-                    "description": "schema",
-                    "examples": {
-                        "application/json": {
-                            "config": {
-                                "features": [
-                                    {
-                                        "method": "POST",
-                                        "path": "/cluster/",
-                                        "available": False,
-                                    },
-                                    {
-                                        "method": "POST",
-                                        "path": "/layout/obs",
-                                        "available": True,
-                                        "interactiveLimit": 10000,
-                                    },
-                                    {
-                                        "method": "POST",
-                                        "path": "/layout/var",
-                                        "available": False,
-                                    },
-                                ],
-                                "displayNames": {
-                                    "engine": "ScanPy version 1.33",
-                                    "dataset": "/home/joe/mouse/blorth.csv",
-                                },
-                            }
-                        }
-                    },
-                }
-            },
-        }
-    )
     def get(self):
         config = {
             "config": {
@@ -158,51 +71,13 @@ class ConfigAPI(Resource):
 
 
 class AnnotationsObsAPI(Resource):
-    @swagger.doc(
-        {
-            "summary": "Fetch annotations (metadata) for all observations.",
-            "tags": ["annotations"],
-            "parameters": [
-                {
-                    "in": "query",
-                    "name": "annotation-name",
-                    "type": "string",
-                    "description": "list of 1 or more annotation names",
-                }
-            ],
-            "responses": {
-                "200": {
-                    "description": "annotations",
-                    "examples": {
-                        "application/json": {
-                            "names": ["tissue_type", "sex", "num_reads", "clusters"],
-                            "data": [
-                                [0, "lung", "F", 39844, 99],
-                                [1, "heart", "M", 83, 1],
-                                [49, "spleen", None, 2, "unknown cluster"],
-                            ],
-                        }
-                    },
-                },
-                "400": {
-                    "description": "one or more of the annotation-name identifiers were not associated with an "
-                    "annotation name"
-                },
-            },
-        }
-    )
     def get(self):
         fields = request.args.getlist("annotation-name", None)
         preferred_mimetype = request.accept_mimetypes.best_match(
-            ["application/json", "application/octet-stream"],
-            "application/json"
+            ["application/octet-stream"]
         )
         try:
-            if preferred_mimetype == "application/json":
-                return make_response(
-                    current_app.data.annotation({}, "obs", fields), HTTPStatus.OK, {"Content-Type": "application/json"}
-                )
-            elif preferred_mimetype == "application/octet-stream":
+            if preferred_mimetype == "application/octet-stream":
                 return make_response(current_app.data.annotation_to_fbs_matrix("obs", fields),
                                      HTTPStatus.OK,
                                      {"Content-Type": "application/octet-stream"})
@@ -210,119 +85,18 @@ class AnnotationsObsAPI(Resource):
                 return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
         except KeyError:
             return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
-        except JSONEncodingValueError as e:
-            # JSON encoding failure, usually due to bad data
-            warnings.warn(JSON_NaN_to_num_warning_msg)
-            return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
-        except ValueError as e:
-            return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
-
-    @swagger.doc(
-        {
-            "summary": "Fetch annotations (metadata) for filtered subset of observations.",
-            "tags": ["annotations"],
-            "parameters": [
-                {
-                    "in": "query",
-                    "name": "annotation-name",
-                    "type": "string",
-                    "description": "list of 1 or more annotation names",
-                },
-                {
-                    "name": "filter",
-                    "description": "Complex Filter",
-                    "in": "body",
-                    "schema": FilterModel,
-                },
-            ],
-            "responses": {
-                "200": {
-                    "description": "annotations",
-                    "examples": {
-                        "application/json": {
-                            "names": ["tissue_type", "sex", "num_reads", "clusters"],
-                            "data": [
-                                [0, "lung", "F", 39844, 99],
-                                [1, "heart", "M", 83, 1],
-                                [49, "spleen", None, 2, "unknown cluster"],
-                            ],
-                        }
-                    },
-                },
-                "400": {
-                    "description": "malformed filter or one or more of the annotation-name identifiers were"
-                    "not associated with an annotation name"
-                },
-            },
-        }
-    )
-    def put(self):
-        fields = request.args.getlist("annotation-name", None)
-        try:
-            annotation_response = current_app.data.annotation(
-                request.get_json()["filter"], "obs", fields
-            )
-            return make_response(
-                annotation_response, HTTPStatus.OK, {"Content-Type": "application/json"}
-            )
-        except KeyError:
-            return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
-        except FilterError as e:
-            return make_response(e.message, HTTPStatus.BAD_REQUEST)
-        except JSONEncodingValueError as e:
-            # JSON encoding failure, usually due to bad data
-            warnings.warn(JSON_NaN_to_num_warning_msg)
-            return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
         except ValueError as e:
             return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
 
 
 class AnnotationsVarAPI(Resource):
-    @swagger.doc(
-        {
-            "summary": "Fetch annotations (metadata) for all variables.",
-            "tags": ["annotations"],
-            "parameters": [
-                {
-                    "in": "query",
-                    "name": "annotation-name",
-                    "type": "string",
-                    "description": "list of 1 or more annotation names",
-                }
-            ],
-            "responses": {
-                "200": {
-                    "description": "annotations",
-                    "examples": {
-                        "application/json": {
-                            "names": ["name", "category"],
-                            "data": [
-                                [0, "ATAD3C", 1],
-                                [1, "RER1", None],
-                                [49, "S100B", 6],
-                            ],
-                        }
-                    },
-                },
-                "400": {
-                    "description": "one or more of the annotation-name identifiers were not associated with an"
-                    " annotation name"
-                },
-            },
-        }
-    )
     def get(self):
         fields = request.args.getlist("annotation-name", None)
         preferred_mimetype = request.accept_mimetypes.best_match(
-            ["application/json", "application/octet-stream"],
-            "application/json"
+            ["application/octet-stream"]
         )
         try:
-            if preferred_mimetype == "application/json":
-                return make_response(current_app.data.annotation({}, "var", fields),
-                                     HTTPStatus.OK,
-                                     {"Content-Type": "application/json"})
-            elif preferred_mimetype == "application/octet-stream":
+            if preferred_mimetype == "application/octet-stream":
                 return make_response(current_app.data.annotation_to_fbs_matrix("var", fields),
                                      HTTPStatus.OK,
                                      {"Content-Type": "application/octet-stream"})
@@ -330,317 +104,22 @@ class AnnotationsVarAPI(Resource):
                 return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
         except KeyError:
             return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
-        except JSONEncodingValueError as e:
-            # JSON encoding failure, usually due to bad data
-            warnings.warn(JSON_NaN_to_num_warning_msg)
-            return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
-        except ValueError as e:
-            return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
-
-    @swagger.doc(
-        {
-            "summary": "Fetch annotations (metadata) for filtered subset of variables.",
-            "tags": ["annotations"],
-            "parameters": [
-                {
-                    "in": "query",
-                    "name": "annotation-name",
-                    "type": "string",
-                    "description": "list of 1 or more annotation names",
-                },
-                {
-                    "name": "filter",
-                    "description": "Complex Filter",
-                    "in": "body",
-                    "schema": FilterModel,
-                },
-            ],
-            "responses": {
-                "200": {
-                    "description": "annotations",
-                    "examples": {
-                        "application/json": {
-                            "names": ["name", "category"],
-                            "data": [
-                                [0, "ATAD3C", 1],
-                                [1, "RER1", None],
-                                [49, "S100B", 6],
-                            ],
-                        }
-                    },
-                },
-                "400": {
-                    "description": "malformed filter or one or more of the annotation-name identifiers were"
-                    "not associated with an annotation name"
-                },
-            },
-        }
-    )
-    def put(self):
-        fields = request.args.getlist("annotation-name", None)
-        try:
-            annotation_response = current_app.data.annotation(
-                request.get_json()["filter"], "var", fields
-            )
-            return make_response(
-                annotation_response, HTTPStatus.OK, {"Content-Type": "application/json"}
-            )
-        except KeyError:
-            return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
-        except FilterError:
-            return make_response("Malformed filter", HTTPStatus.BAD_REQUEST)
-        except JSONEncodingValueError as e:
-            # JSON encoding failure, usually due to bad data
-            warnings.warn(JSON_NaN_to_num_warning_msg)
-            return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
-        except ValueError as e:
-            return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
-
-
-class DataObsAPI(Resource):
-    @swagger.doc(
-        {
-            "summary": "Get data (expression values) from the dataframe.",
-            "tags": ["data"],
-            "parameters": [
-                {
-                    "in": "query",
-                    "name": "filter",
-                    "type": "string",
-                    "description": "axis:key:value",
-                },
-                {
-                    "in": "query",
-                    "name": "accept-type",
-                    "type": "string",
-                    "description": "MIME type",
-                },
-            ],
-            "responses": {
-                "200": {
-                    "description": "expression",
-                    "examples": {
-                        "application/json": {
-                            "var": [0, 20000],
-                            "obs": [[1, 39483, 3902, 203, 0, 0, 28]],
-                        }
-                    },
-                },
-                "400": {"description": "Malformed filter"},
-                "406": {"description": "Unacceptable MIME type"},
-            },
-        }
-    )
-    def get(self):
-        accept_type = request.args.get("accept-type", None)
-        # request.args is immutable
-        args = request.args.copy()
-        args.pop("accept-type", None)
-        try:
-            filter_ = parse_filter(
-                ImmutableMultiDict(args), current_app.data.schema["annotations"]
-            )
-        except QueryStringError as e:
-            return make_response(e.message, HTTPStatus.BAD_REQUEST)
-        # TODO support CSV
-        try:
-            # TODO store mime_type when more than one is supported
-            get_mime_type(
-                acceptable_types=["application/json"],
-                query_param=accept_type,
-                header=request.accept_mimetypes,
-            )
-        except MimeTypeError as e:
-            return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
-        try:
-            return make_response(
-                current_app.data.data_frame(filter_, axis=Axis.OBS),
-                HTTPStatus.OK,
-                {"Content-Type": "application/json"},
-            )
-        except FilterError as e:
-            return make_response(e.message, HTTPStatus.BAD_REQUEST)
-        except JSONEncodingValueError as e:
-            # JSON encoding failure, usually due to bad data
-            warnings.warn(JSON_NaN_to_num_warning_msg)
-            return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
-        except ValueError as e:
-            return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
-
-    @swagger.doc(
-        {
-            "summary": "Get data (expression values) from the dataframe.",
-            "tags": ["data"],
-            "parameters": [
-                {
-                    "name": "filter",
-                    "description": "Complex Filter",
-                    "in": "body",
-                    "schema": FilterModel,
-                }
-            ],
-            "responses": {
-                "200": {
-                    "description": "expression",
-                    "examples": {
-                        "application/json": {
-                            "var": [0, 20000],
-                            "obs": [[1, 39483, 3902, 203, 0, 0, 28]],
-                        }
-                    },
-                },
-                "400": {"description": "Malformed filter"},
-                "406": {"description": "Unacceptable MIME type"},
-            },
-        }
-    )
-    def put(self):
-        if not request.accept_mimetypes.best_match(["application/json", "text/csv"]):
-            return make_response(
-                f"Unsupported MIME type '{request.accept_mimetypes}'",
-                HTTPStatus.NOT_ACCEPTABLE,
-            )
-        try:
-            get_mime_type(
-                acceptable_types=["application/json"], header=request.accept_mimetypes
-            )
-        except MimeTypeError as e:
-            return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
-        try:
-            return make_response(
-                (
-                    current_app.data.data_frame(
-                        request.get_json()["filter"], axis=Axis.OBS
-                    )
-                ),
-                HTTPStatus.OK,
-                {"Content-Type": "application/json"},
-            )
-        except FilterError as e:
-            return make_response(e.message, HTTPStatus.BAD_REQUEST)
-        except JSONEncodingValueError as e:
-            # JSON encoding failure, usually due to bad data
-            warnings.warn(JSON_NaN_to_num_warning_msg)
-            return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
         except ValueError as e:
             return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
 
 
 class DataVarAPI(Resource):
-    @swagger.doc(
-        {
-            "summary": "Get data (expression values) from the dataframe.",
-            "tags": ["data"],
-            "parameters": [
-                {
-                    "in": "query",
-                    "name": "filter",
-                    "type": "string",
-                    "description": "axis:key:value",
-                },
-                {
-                    "in": "query",
-                    "name": "accept-type",
-                    "type": "string",
-                    "description": "MIME type",
-                },
-            ],
-            "responses": {
-                "200": {
-                    "description": "expression",
-                    "examples": {
-                        "application/json": {
-                            "obs": [0, 20000],
-                            "var": [[1, 39483, 3902, 203, 0, 0, 28]],
-                        }
-                    },
-                },
-                "400": {"description": "Malformed filter"},
-                "406": {"description": "Unacceptable MIME type"},
-            },
-        }
-    )
-    def get(self):
-        accept_type = request.args.get("accept-type", None)
-        # request.args is immutable
-        args = request.args.copy()
-        args.pop("accept-type", None)
-        try:
-            filter_ = parse_filter(
-                ImmutableMultiDict(args), current_app.data.schema["annotations"]
-            )
-        except QueryStringError as e:
-            return make_response(e.message, HTTPStatus.BAD_REQUEST)
-        try:
-            get_mime_type(
-                acceptable_types=["application/json"],
-                query_param=accept_type,
-                header=request.accept_mimetypes,
-            )
-        except MimeTypeError as e:
-            return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
-        try:
-            return make_response(
-                current_app.data.data_frame(filter_, axis=Axis.VAR),
-                HTTPStatus.OK,
-                {"Content-Type": "application/json"},
-            )
-        except FilterError as e:
-            return make_response(e.message, HTTPStatus.BAD_REQUEST)
-        except JSONEncodingValueError as e:
-            # JSON encoding failure, usually due to bad data
-            warnings.warn(JSON_NaN_to_num_warning_msg)
-            return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
-        except ValueError as e:
-            return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
-
-    @swagger.doc(
-        {
-            "summary": "Get data (expression values) from the dataframe.",
-            "tags": ["data"],
-            "parameters": [
-                {
-                    "name": "filter",
-                    "description": "Complex Filter",
-                    "in": "body",
-                    "schema": FilterModel,
-                }
-            ],
-            "responses": {
-                "200": {
-                    "description": "expression",
-                    "examples": {
-                        "application/json": {
-                            "obs": [0, 20000],
-                            "var": [[1, 39483, 3902, 203, 0, 0, 28]],
-                        }
-                    },
-                },
-                "400": {"description": "Malformed filter"},
-                "406": {"description": "Unacceptable MIME type"},
-            },
-        }
-    )
     def put(self):
         preferred_mimetype = request.accept_mimetypes.best_match(
-            ["application/json", "application/octet-stream"],
-            "application/json"
+            ["application/octet-stream"]
         )
         try:
-            if preferred_mimetype == "application/json":
-                return make_response(
-                    (
-                        current_app.data.data_frame(
-                            request.get_json()["filter"], axis=Axis.VAR
-                        )
-                    ),
-                    HTTPStatus.OK,
-                    {"Content-Type": "application/json"},
-                )
-            elif preferred_mimetype == "application/octet-stream":
+            if preferred_mimetype == "application/octet-stream":
+                filter_json = request.get_json()
+                filter = filter_json["filter"] if filter_json else None
                 return make_response(
                     current_app.data.data_frame_to_fbs_matrix(
-                        request.get_json()["filter"], axis=Axis.VAR
+                        filter, axis=Axis.VAR
                     ),
                     HTTPStatus.OK,
                     {"Content-Type": "application/octet-stream"})
@@ -648,73 +127,11 @@ class DataVarAPI(Resource):
                 return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
         except FilterError as e:
             return make_response(e.message, HTTPStatus.BAD_REQUEST)
-        except JSONEncodingValueError as e:
-            # JSON encoding failure, usually due to bad data
-            warnings.warn(JSON_NaN_to_num_warning_msg)
-            return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
         except ValueError as e:
             return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
 
 
 class DiffExpObsAPI(Resource):
-    @swagger.doc(
-        {
-            "summary": "Generate differential expression (DE) statistics for two specified subsets of data, "
-            "as indicated by the two provided observation complex filters",
-            "tags": ["diffexp"],
-            # TODO sort out params
-            # "parameters": [
-            #     # {
-            #     #     "in": "body",
-            #     #     "name": "mode",
-            #     #     "type": "string",
-            #     #     "required": True,
-            #     #     "description": "topN or varFilter"
-            #     # },
-            #     {
-            #         "in": "query",
-            #         "name": "count",
-            #         "type": "int32",
-            #         "description": "TopN mode: how many vars to return"
-            #     },
-            #     {
-            #         "in": "body",
-            #         "name": "varFilter",
-            #         "schema": FilterModel,
-            #         "description": "varFilter: Complex filter, only var for which vars to return"
-            #     },
-            #     {
-            #         "in": "body",
-            #         "name": "set1",
-            #         "schema": FilterModel,
-            #         "required": True,
-            #         "description": "Complex filter, only obs - observations in set1"
-            #     },
-            #     {
-            #         "in": "body",
-            #         "name": "set2",
-            #         "schema": FilterModel,
-            #         "description": "Complex filter, only obs - observations in set2.
-            # If not included, inverse of set1."
-            #     },
-            # ],
-            "responses": {
-                "200": {
-                    "description": "Statistics are encoded as an array of arrays, with fields ordered as: "
-                    "varIndex, logfoldchange,  pVal, pValAdj",
-                    "examples": {
-                        "application/json": [
-                            [328, -2.569_489, 2.655_706e-63, 3.642_036e-57],
-                            [1250, -2.569_489, 2.655_706e-63, 3.642_036e-57],
-                        ]
-                    },
-                },
-                "400": {"description": "malformed filter"},
-                "403": {"description": "non-interactive request"},
-                "501": {"description": "diffexp is not implemented"},
-            },
-        }
-    )
     def post(self):
         args = request.get_json()
         # confirm mode is present and legal
@@ -783,40 +200,12 @@ class DiffExpObsAPI(Resource):
 
 
 class LayoutObsAPI(Resource):
-    @swagger.doc(
-        {
-            "summary": "Get the default layout for all observations.",
-            "tags": ["layout"],
-            "parameters": [],
-            "responses": {
-                "200": {
-                    "description": "layout",
-                    "examples": {
-                        "application/json": {
-                            "layout": {
-                                "ndims": 2,
-                                "coordinates": [
-                                    [0, 0.284_483, 0.983_744],
-                                    [1, 0.038_844, 0.739_444],
-                                ],
-                            }
-                        }
-                    },
-                },
-                "400": {"description": "Data preparation error"},
-            },
-        }
-    )
     def get(self):
         preferred_mimetype = request.accept_mimetypes.best_match(
-            ["application/json", "application/octet-stream"],
-            "application/json"
+            ["application/octet-stream"]
         )
         try:
-            if preferred_mimetype == "application/json":
-                return make_response(current_app.data.layout({}), HTTPStatus.OK, {"Content-Type": "application/json"})
-
-            elif preferred_mimetype == "application/octet-stream":
+            if preferred_mimetype == "application/octet-stream":
                 return make_response(current_app.data.layout_to_fbs_matrix(),
                                      HTTPStatus.OK,
                                      {"Content-Type": "application/octet-stream"})
@@ -824,69 +213,19 @@ class LayoutObsAPI(Resource):
                 return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
         except PrepareError as e:
             return make_response(e.message, HTTPStatus.INTERNAL_SERVER_ERROR)
-        except JSONEncodingValueError as e:
-            # JSON encoding failure, usually due to bad data
-            warnings.warn(JSON_NaN_to_num_warning_msg)
-            return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
         except ValueError as e:
             return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
 
-    # @swagger.doc({
-    #     "summary": "Observation layout for filtered subset.",
-    #     "tags": ["layout"],
-    #     "parameters": [
-    #         {
-    #             "name": "filter",
-    #             "description": "Complex Filter",
-    #             "in": "body",
-    #             "schema": FilterModel
-    #         }
-    #     ],
-    #     "responses": {
-    #         "200": {
-    #             "description": "layout",
-    #             "examples": {
-    #                 "application/json": {
-    #                     "layout": {
-    #                         "ndims": 2,
-    #                         "coordinates": [
-    #                             [0, 0.284483, 0.983744],
-    #                             [1, 0.038844, 0.739444]
-    #                         ]
-    #                     }
-    #                 }
-    #             }
-    #         },
-    #         "400": {
-    #             "description": "Malformed filter"
-    #         },
-    #         "403": {
-    #             "description": "Non-interactive request"
-    #         },
-    #     }
-    # })
-    # def put(self):
-    #     try:
-    #         filter = request.get_json()["filter"]
-    #         interactive_limit = current_app.data.features["layout"]["obs"]["interactiveLimit"]
-    #         layout = current_app.data.layout(filter, interactive_limit=interactive_limit)
-    #         return make_response(layout, HTTPStatus.OK, {"Content-Type": content_type})
-    #     except FilterError as e:
-    #         return make_response(e.message, HTTPStatus.BAD_REQUEST)
-    #     except InteractiveError:
-    #         return make_response("Non-interactive request", HTTPStatus.FORBIDDEN)
-
 
 def get_api_resources():
     bp = Blueprint("api", __name__, url_prefix="/api/v0.2")
-    api = Api(bp, add_api_spec_resource=False)
+    api = Api(bp)
     # Initialization routes
     api.add_resource(SchemaAPI, "/schema")
     api.add_resource(ConfigAPI, "/config")
     # Data routes
     api.add_resource(AnnotationsObsAPI, "/annotations/obs")
     api.add_resource(AnnotationsVarAPI, "/annotations/var")
-    api.add_resource(DataObsAPI, "/data/obs")
     api.add_resource(DataVarAPI, "/data/var")
     # Computation routes
     api.add_resource(DiffExpObsAPI, "/diffexp/obs")

server/app/scanpy_engine/scanpy_engine.py

@@ -1,16 +1,13 @@
 import warnings
 
 import numpy as np
-from pandas import DataFrame
 from pandas.core.dtypes.dtypes import CategoricalDtype
 import scanpy.api as sc
-from scipy import sparse
 
 from server.app.driver.driver import CXGDriver
 from server.app.util.constants import Axis, DEFAULT_TOP_N
 from server.app.util.errors import (
     FilterError,
-    InteractiveError,
     JSONEncodingValueError,
     PrepareError,
     ScanpyFileError,
@@ -197,23 +194,6 @@ class ScanpyEngine(CXGDriver):
                 f"to solve this problem. "
             )
 
-    def filter_dataframe(self, filter):
-        """
-         Filter cells from data and return a subset of the data. They can operate on both obs and var dimension with
-         indexing and filtering by annotation value. Filters are combined with the and operator.
-         See REST specs for info on filter format:
-         # TODO update this link to swagger when it's done
-         https://docs.google.com/document/d/1Fxjp1SKtCk7l8QP9-7KAjGXL0eldi_qEnNT0NmlGzXI/edit#heading=h.8qc9q57amldx
-
-        :param filter: dictionary with filter params
-        :return: View into scanpy object with cells/genes filtered
-        """
-        if not filter:
-            return self.data
-        obs_selector, var_selector = self._filter_to_mask(filter, use_slices=False)
-        data = self._slice(self.data, obs_selector, var_selector)
-        return data
-
     @staticmethod
     def _annotation_filter_to_mask(filter, d_axis, count):
         mask = np.ones((count,), dtype=bool)
@@ -277,72 +257,6 @@ class ScanpyEngine(CXGDriver):
                 )
         return obs_selector, var_selector
 
-    @staticmethod
-    def _slice(data, obs_selector=None, vars_selector=None):
-        """
-        Slice date using any selector that the AnnData object
-        supprots for slicing.  If selector is None, will not slice
-        on that axis.
-
-        This method exists to optimize filtering/slicing sparse data that has
-        access patterns which impact slicing performance.
-
-        https://docs.scipy.org/doc/scipy/reference/sparse.html
-        """
-        prefer_row_access = (
-            sparse.isspmatrix_csr(data._X)
-            or sparse.isspmatrix_lil(data._X)
-            or sparse.isspmatrix_bsr(data._X)
-        )
-        if prefer_row_access:
-            # Row-major slicing
-            if obs_selector is not None:
-                data = data[obs_selector, :]
-            if vars_selector is not None:
-                data = data[:, vars_selector]
-        else:
-            # Col-major slicing
-            if vars_selector is not None:
-                data = data[:, vars_selector]
-            if obs_selector is not None:
-                data = data[obs_selector, :]
-
-        return data
-
-    def annotation(self, filter, axis, fields=None):
-        """
-         Gets annotation value for each observation
-        :param filter: filter: dictionary with filter params
-        :param axis: string obs or var
-        :param fields: list of keys for annotation to return, returns all annotation values if not set.
-        :return: dict: names - list of fields in order, data - list of lists or metadata
-        [observation ids, val1, val2...]
-        """
-        try:
-            obs_selector, var_selector = self._filter_to_mask(filter)
-        except (KeyError, IndexError) as e:
-            raise FilterError(f"Error parsing filter: {e}") from e
-        if axis == Axis.OBS:
-            obs = self.data.obs[obs_selector]
-            if not fields:
-                fields = obs.columns.tolist()
-            result = {
-                "names": fields,
-                "data": DataFrame(obs[fields]).to_records(index=True).tolist(),
-            }
-        else:
-            var = self.data.var[var_selector]
-            if not fields:
-                fields = var.columns.tolist()
-            result = {
-                "names": fields,
-                "data": DataFrame(var[fields]).to_records(index=True).tolist(),
-            }
-        try:
-            return jsonify_scanpy(result)
-        except ValueError:
-            raise JSONEncodingValueError("Error encoding annotations to JSON")
-
     def annotation_to_fbs_matrix(self, axis, fields=None):
         if axis == Axis.OBS:
             df = self.data.obs
@@ -352,44 +266,6 @@ class ScanpyEngine(CXGDriver):
             df = df[fields]
         return encode_matrix_fbs(df, col_idx=df.columns)
 
-    def data_frame(self, filter, axis):
-        """
-        Retrieves data for each variable for observations in data frame
-        :param filter: filter: dictionary with filter params
-        :param axis: string obs or var
-        :return: {
-            "var": list of variable ids,
-            "obs": [cellid, var1 expression, var2 expression, ...],
-        }
-        """
-        try:
-            obs_selector, var_selector = self._filter_to_mask(filter)
-        except (KeyError, IndexError) as e:
-            raise FilterError(f"Error parsing filter: {e}") from e
-        _X = self.data._X[obs_selector, var_selector]
-        if sparse.issparse(_X):
-            _X = _X.toarray()
-        var_index_sliced = self.data.var.index[var_selector]
-        obs_index_sliced = self.data.obs.index[obs_selector]
-        if axis == Axis.OBS:
-            result = {
-                "var": var_index_sliced.tolist(),
-                "obs": DataFrame(_X, index=obs_index_sliced)
-                .to_records(index=True)
-                .tolist(),
-            }
-        else:
-            result = {
-                "obs": obs_index_sliced.tolist(),
-                "var": DataFrame(_X.T, index=var_index_sliced)
-                .to_records(index=True)
-                .tolist(),
-            }
-        try:
-            return jsonify_scanpy(result)
-        except ValueError:
-            raise JSONEncodingValueError("Error encoding dataframe to JSON")
-
     def data_frame_to_fbs_matrix(self, filter, axis):
         """
         Retrieves data 'X' and returns in a flatbuffer Matrix.
@@ -405,7 +281,7 @@ class ScanpyEngine(CXGDriver):
             raise ValueError("Only VAR dimension access is supported")
         try:
             obs_selector, var_selector = self._filter_to_mask(filter, use_slices=False)
-        except (KeyError, IndexError) as e:
+        except (KeyError, IndexError, TypeError) as e:
             raise FilterError(f"Error parsing filter: {e}") from e
         if obs_selector is not None:
             raise FilterError("filtering on obs unsupported")
@@ -440,50 +316,6 @@ class ScanpyEngine(CXGDriver):
                 "Error encoding differential expression to JSON"
             )
 
-    def layout(self, filter, interactive_limit=None):
-        """
-        Computes a n-d layout for cells through dimensionality reduction.
-        :param filter: filter: dictionary with filter params
-        :param interactive_limit: -- don't compute if total # genes in dataframes are larger than this
-        :return:  [cellid, x, y, ...]
-        """
-        try:
-            df = self.filter_dataframe(filter)
-        except (KeyError, IndexError) as e:
-            raise FilterError(f"Error parsing filter: {e}") from e
-        if interactive_limit and len(df.obs.index) > interactive_limit:
-            raise InteractiveError("Size data is too large for interactive computation")
-        # TODO Filtering cells is fine, but filtering genes does nothing because the neighbors are
-        # calculated using the original vars (geneset) and this doesn’t get updated when you use less.
-        # Need to recalculate neighbors (long) if user requests new layout filtered by var
-        # TODO for MVP we are pushing computation of layout to preprocessing and not allowing re-layout
-        # this will probably change after user feedback
-        # getattr(sc.tl, self.layout_method)(df, random_state=123)
-        try:
-            df_layout = df.obsm[f"X_{self.layout_method}"]
-        except ValueError as e:
-            raise PrepareError(
-                f"Layout has not been calculated using {self.layout_method}, "
-                f"please prepare your datafile and relaunch cellxgene"
-            ) from e
-        normalized_layout = DataFrame(
-            (df_layout - df_layout.min()) / (df_layout.max() - df_layout.min()),
-            index=df.obs.index,
-        )
-        try:
-            return jsonify_scanpy(
-                {
-                    "layout": {
-                        "ndims": normalized_layout.shape[1],
-                        "coordinates": normalized_layout.to_records(
-                            index=True
-                        ).tolist(),
-                    }
-                }
-            )
-        except ValueError:
-            raise JSONEncodingValueError("Error encoding layout to JSON")
-
     def layout_to_fbs_matrix(self):
         """
         Return the default 2-D layout for cells as a FBS Matrix.

server/app/util/filter.py

@@ -1,86 +0,0 @@
-import json
-from collections import defaultdict
-
-from numpy import float32, int32
-
-from server.app.util.constants import Axis
-
-
-class QueryStringError(Exception):
-    def __init__(self, key, message):
-        self.key = key
-        self.message = message
-
-
-def _convert_variable(datatype, variable):
-    """
-    Convert variable to number (float/int)
-    Used for dataset metadata and for query string
-    :param datatype: type to convert to
-    :param variable (string or None): value of variable
-    :return: converted variable
-    :raises: AssertionError
-    """
-    assert datatype in ["boolean", "categorical", "float32", "int32", "string"]
-    if variable is None:
-        return variable
-    if datatype == "int32":
-        variable = int32(variable)
-    elif datatype == "float32":
-        variable = float32(variable)
-    elif datatype == "boolean":
-        variable = json.loads(variable)
-        assert isinstance(variable, bool)
-    return variable
-
-
-def parse_filter(query_filter, schema):
-    """
-    The filter comes in as arguments from a GET request
-    For categorical metadata keys filter based on axis:key=value
-    For continuous metadata keys filter by axis:key=min,max
-    Either value can be replaced by a * To have only a minimum
-    value axis:key=min,* To have only a maximum value axis:key=*,max
-
-    They combine via AND so a cell's metadata would have to match every filter
-
-    The results is a matrix with the cells the pass the filter and at this point all the genes
-    :param query_filter: flask's request.args
-    :param schema: dictionary schema
-    :raises QueryStringError
-    :return:
-    """
-    query = defaultdict(lambda: defaultdict(list))
-
-    for key in query_filter:
-        axis, annotation = key.split(":", 1)
-        try:
-            Axis(axis)
-        except ValueError:
-            raise QueryStringError(key, f"Error: key {key} not in metadata schema")
-        ann_filter = {"name": annotation}
-        for ann in schema[axis]:
-            if ann["name"] == annotation:
-                dtype = ann["type"]
-                break
-        else:
-            raise QueryStringError(key, f"Error: {annotation} not a valid annotation name")
-        if dtype in ["string", "categorical", "boolean"]:
-            ann_filter["values"] = [_convert_variable(dtype, i) for i in query_filter.getlist(key)]
-        else:
-            value = query_filter.get(key)
-            try:
-                min_, max_ = value.split(",")
-            except ValueError:
-                raise QueryStringError(key, f"Error: min,max format required for range for {annotation}, got {value}")
-            if min_ == "*":
-                min_ = None
-            if max_ == "*":
-                max_ = None
-            try:
-                ann_filter["min"] = _convert_variable(dtype, min_)
-                ann_filter["max"] = _convert_variable(dtype, max_)
-            except ValueError:
-                raise QueryStringError(key, f"Error: expected type {query[key]['type']} for key {key}, got {value}")
-        query[axis]["annotation_value"].append(ann_filter)
-    return query

server/app/util/models.py

@@ -1,34 +0,0 @@
-from flask_restful_swagger_2 import Schema
-
-
-class AnnotationModel(Schema):
-    type = "object"
-    description = "Filter by annotation key: value"
-    properties = {
-        "name": {"type": "string"},
-        # TODO update to OpenAPI v3.0 when a library is available that supports it
-        # Unfortunately 2.0 doesn't have a way to have a schema that accepts multiple types
-        # Overloading the type key with a list seems to work ok and makes it to the page
-        "values": {"type": "array", "items": {"type": ["float32", "string", "int32", "bool"]}},
-        "min": {"type": ["int32", "float32"]},
-        "max": {"type": ["int32", "float32"]},
-    }
-    required = ["name"]
-
-
-class IndexModel(Schema):
-    type = "object"
-    description = "Filter by index of observation/variable ex. [0, 5, 15]"
-    properties = {"index": {"type": "array", "items": {"format": "int32", "type": "integer"}}}
-
-
-class AxisModel(Schema):
-    type = "object"
-    description = "Axis of data -- obs or var"
-    properties = {"index": IndexModel, "annotation_value": AnnotationModel.array()}
-
-
-class FilterModel(Schema):
-    type = "object"
-    description = "Complex filter"
-    properties = {"filter": {"type": "object", "properties": {"obs": AxisModel, "var": AxisModel}}}

server/app/util/utils.py

@@ -1,10 +1,6 @@
 import json
-from argparse import ArgumentTypeError
-
 from numpy import float32, integer
 
-from server.app.util.errors import MimeTypeError
-
 
 class Float32JSONEncoder(json.JSONEncoder):
     def __init__(self, *args, **kwargs):
@@ -30,31 +26,5 @@ def custom_format_warning(msg, *args, **kwargs):
     return f"[cellxgene] Warning: {msg} \n"
 
 
-def get_mime_type(
-    default="application/json", acceptable_types=["application/json", "text/csv"], query_param=None, header=None
-):
-    mime_type = default
-    if query_param:
-        if query_param in acceptable_types:
-            mime_type = query_param
-        else:
-            raise MimeTypeError(f"Unsupported mime type {query_param} specified in query parameter 'accept-type'")
-    elif len(header):
-        mime_type = header.best_match(acceptable_types)
-        if not mime_type:
-            raise MimeTypeError(f"Unsupported mime type(s) {header} in HTTP Accept header")
-    return mime_type
-
-
-def whole_number(value):
-    try:
-        value = int(value)
-    except ValueError as e:
-        raise ArgumentTypeError(f"{value} is not type int") from e
-    if value < 0:
-        raise ArgumentTypeError(f"{value} is not >= 0")
-    return value
-
-
 def jsonify_scanpy(data):
     return json.dumps(data, cls=Float32JSONEncoder, allow_nan=False)

server/app/web/templates/swagger.html

@@ -1,97 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-	<meta charset="UTF-8">
-	<title>cellxgene REST API - Swagger definition</title>
-	<link href="https://fonts.googleapis.com/css?family=Open+Sans:400,700|Source+Code+Pro:300,600|Titillium+Web:400,600,700"
-		  rel="stylesheet">
-	<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.2.1/swagger-ui.css"
-		  crossorigin="anonymous"/>
-	<style>
-		html {
-			box-sizing: border-box;
-			overflow: -moz-scrollbars-vertical;
-			overflow-y: scroll;
-		}
-
-		*,
-		*:before,
-		*:after {
-			box-sizing: inherit;
-		}
-
-		body {
-			margin: 0;
-			background: #fafafa;
-		}
-	</style>
-</head>
-
-<body>
-
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"
-	 style="position:absolute;width:0;height:0">
-	<defs>
-		<symbol viewBox="0 0 20 20" id="unlocked">
-			<path d="M15.8 8H14V5.6C14 2.703 12.665 1 10 1 7.334 1 6 2.703 6 5.6V6h2v-.801C8 3.754 8.797 3 10 3c1.203 0 2 .754 2 2.199V8H4c-.553 0-1 .646-1 1.199V17c0 .549.428 1.139.951 1.307l1.197.387C5.672 18.861 6.55 19 7.1 19h5.8c.549 0 1.428-.139 1.951-.307l1.196-.387c.524-.167.953-.757.953-1.306V9.199C17 8.646 16.352 8 15.8 8z"></path>
-		</symbol>
-
-		<symbol viewBox="0 0 20 20" id="locked">
-			<path d="M15.8 8H14V5.6C14 2.703 12.665 1 10 1 7.334 1 6 2.703 6 5.6V8H4c-.553 0-1 .646-1 1.199V17c0 .549.428 1.139.951 1.307l1.197.387C5.672 18.861 6.55 19 7.1 19h5.8c.549 0 1.428-.139 1.951-.307l1.196-.387c.524-.167.953-.757.953-1.306V9.199C17 8.646 16.352 8 15.8 8zM12 8H8V5.199C8 3.754 8.797 3 10 3c1.203 0 2 .754 2 2.199V8z"/>
-		</symbol>
-
-		<symbol viewBox="0 0 20 20" id="close">
-			<path d="M14.348 14.849c-.469.469-1.229.469-1.697 0L10 11.819l-2.651 3.029c-.469.469-1.229.469-1.697 0-.469-.469-.469-1.229 0-1.697l2.758-3.15-2.759-3.152c-.469-.469-.469-1.228 0-1.697.469-.469 1.228-.469 1.697 0L10 8.183l2.651-3.031c.469-.469 1.228-.469 1.697 0 .469.469.469 1.229 0 1.697l-2.758 3.152 2.758 3.15c.469.469.469 1.229 0 1.698z"/>
-		</symbol>
-
-		<symbol viewBox="0 0 20 20" id="large-arrow">
-			<path d="M13.25 10L6.109 2.58c-.268-.27-.268-.707 0-.979.268-.27.701-.27.969 0l7.83 7.908c.268.271.268.709 0 .979l-7.83 7.908c-.268.271-.701.27-.969 0-.268-.269-.268-.707 0-.979L13.25 10z"/>
-		</symbol>
-
-		<symbol viewBox="0 0 20 20" id="large-arrow-down">
-			<path d="M17.418 6.109c.272-.268.709-.268.979 0s.271.701 0 .969l-7.908 7.83c-.27.268-.707.268-.979 0l-7.908-7.83c-.27-.268-.27-.701 0-.969.271-.268.709-.268.979 0L10 13.25l7.418-7.141z"/>
-		</symbol>
-
-
-		<symbol viewBox="0 0 24 24" id="jump-to">
-			<path d="M19 7v4H5.83l3.58-3.59L8 6l-6 6 6 6 1.41-1.41L5.83 13H21V7z"/>
-		</symbol>
-
-		<symbol viewBox="0 0 24 24" id="expand">
-			<path d="M10 18h4v-2h-4v2zM3 6v2h18V6H3zm3 7h12v-2H6v2z"/>
-		</symbol>
-
-	</defs>
-</svg>
-
-<div id="swagger-ui"></div>
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.2.1/swagger-ui-bundle.js"
-		 crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.2.1/swagger-ui-standalone-preset.js"
-		crossorigin="anonymous"></script>
-<script>
-	window.onload = function () {
-		const ui = SwaggerUIBundle({
-			url: window.location.href.replace(/\/swagger$/, "") + "/api/swagger.json",
-			dom_id: '#swagger-ui',
-			deepLinking: true,
-			presets: [
-				SwaggerUIBundle.presets.apis,
-				SwaggerUIStandalonePreset
-			],
-			plugins: [
-				SwaggerUIBundle.plugins.DownloadUrl
-			],
-			layout: "StandaloneLayout"
-		});
-
-		window.ui = ui
-	}
-</script>
-</body>
-
-</html>
-
-
-

server/app/web/webapp.py

@@ -11,13 +11,6 @@ def index():
     return render_template("index.html", datasetTitle=dataset_title)
 
 
-# renders swagger documentation
-@bp.route("/swagger")
-def swag():
-    return render_template("swagger.html")
-
-
-# renders swagger documentation
 @bp.route("/favicon.png")
 def favicon():
     return send_from_directory(os.path.join(bp.root_path, "static/img/"), "favicon.png")

server/cli/cli.py

@@ -5,7 +5,7 @@ from .prepare import prepare
 
 
 @click.group(name="cellxgene", context_settings=dict(max_content_width=85))
-@click.version_option(version="0.6.1", prog_name="cellxgene", message="[%(prog)s] Version %(version)s")
+@click.version_option(version="0.7.0", prog_name="cellxgene", message="[%(prog)s] Version %(version)s")
 def cli():
     pass
 

server/requirements.txt

@@ -5,7 +5,6 @@ Flask-Caching>=1.4.0
 Flask-Compress>=1.4.0
 Flask-Cors>=3.0.6
 Flask-RESTful>=0.3.6
-flask-restful-swagger-2>=0.35
 flatbuffers>=1.10.0
 matplotlib>=2.2
 numpy>=1.14.5

server/test/test_api.py

@@ -57,16 +57,6 @@ class EndPoints(unittest.TestCase):
         self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k")
         self.assertEqual(len(result_data["config"]["features"]), 4)
 
-    def test_get_layout(self):
-        endpoint = "layout/obs"
-        url = f"{URL_BASE}{endpoint}"
-        result = self.session.get(url)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data = result.json()
-        self.assertEqual(result_data["layout"]["ndims"], 2)
-        self.assertEqual(len(result_data["layout"]["coordinates"]), 2638)
-
     def test_get_layout_fbs(self):
         endpoint = "layout/obs"
         url = f"{URL_BASE}{endpoint}"
@@ -82,53 +72,11 @@ class EndPoints(unittest.TestCase):
         self.assertIsNone(df['row_idx'])
         self.assertEqual(len(df['columns']), df['n_cols'])
 
-    # def test_put_layout(self):
-    #     endpoint = "layout/obs"
-    #     url = f"{URL_BASE}{endpoint}"
-    #     obs_filter = {
-    #         "filter": {
-    #             "obs": {
-    #                 "annotation_value": [
-    #                     {"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
-    #                     {"name": "n_counts", "min": 3000},
-    #                 ],
-    #                 "index": [1, 99, [1000, 2000]]
-    #             }
-    #         }
-    #     }
-    #     result = self.session.put(url, json=obs_filter)
-    #     self.assertEqual(result.status_code, HTTPStatus.OK)
-    #     result_data = result.json()
-    #     self.assertEqual(len(result_data["layout"]["coordinates"]), 15)
-
     def test_bad_filter(self):
-        endpoints = ["annotations/obs", "annotations/var", "data/obs", "data/var"]
-        for endpoint in endpoints:
-            url = f"{URL_BASE}{endpoint}"
-            result = self.session.put(url, json=BAD_FILTER)
-            self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
-
-    def test_get_annotations_obs(self):
-        endpoint = "annotations/obs"
+        endpoint = "data/var"
         url = f"{URL_BASE}{endpoint}"
-        result = self.session.get(url)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data = result.json()
-        self.assertEqual(result_data["names"], ["name", "n_genes", "percent_mito", "n_counts", "louvain"])
-        self.assertEqual(len(result_data["data"]), 2638)
-        self.assertEqual(len(result_data["data"][0]), 6)
-
-    def test_get_annotations_obs_keys(self):
-        endpoint = "annotations/obs"
-        query = "annotation-name=n_genes&annotation-name=percent_mito"
-        url = f"{URL_BASE}{endpoint}?{query}"
-        result = self.session.get(url)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        result_data = result.json()
-        self.assertEqual(result_data["names"], ["n_genes", "percent_mito"])
-        self.assertEqual(len(result_data["data"][0]), 3)
+        result = self.session.put(url, json=BAD_FILTER)
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
 
     def test_get_annotations_obs_fbs(self):
         endpoint = "annotations/obs"
@@ -146,6 +94,23 @@ class EndPoints(unittest.TestCase):
         self.assertEqual(len(df['columns']), df['n_cols'])
         self.assertListEqual(df['col_idx'], ['name', 'n_genes', 'percent_mito', 'n_counts', 'louvain'])
 
+    def test_get_annotations_obs_keys_fbs(self):
+        endpoint = "annotations/obs"
+        query = "annotation-name=n_genes&annotation-name=percent_mito"
+        url = f"{URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df['n_rows'], 2638)
+        self.assertEqual(df['n_cols'], 2)
+        self.assertIsNotNone(df['columns'])
+        self.assertIsNotNone(df['col_idx'])
+        self.assertIsNone(df['row_idx'])
+        self.assertEqual(len(df['columns']), df['n_cols'])
+        self.assertListEqual(df['col_idx'], ['n_genes', 'percent_mito'])
+
     def test_get_annotations_obs_error(self):
         endpoint = "annotations/obs"
         query = "annotation-name=notakey"
@@ -153,50 +118,6 @@ class EndPoints(unittest.TestCase):
         result = self.session.get(url)
         self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
 
-    def test_put_annotations_obs(self):
-        endpoint = "annotations/obs"
-        url = f"{URL_BASE}{endpoint}"
-        obs_filter = {
-            "filter": {
-                "obs": {
-                    "annotation_value": [
-                        {"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
-                        {"name": "n_counts", "min": 3000},
-                    ],
-                    "index": [1, 99, [1000, 2000]],
-                }
-            }
-        }
-        result = self.session.put(url, json=obs_filter)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data = result.json()
-        self.assertEqual(result_data["names"], ["name", "n_genes", "percent_mito", "n_counts", "louvain"])
-        self.assertEqual(len(result_data["data"]), 15)
-
-    def test_filter_put_annotations_obs(self):
-        endpoint = "annotations/obs"
-        query = "annotation-name=n_genes&annotation-name=percent_mito"
-        url = f"{URL_BASE}{endpoint}?{query}"
-        obs_filter = {
-            "filter": {
-                "obs": {
-                    "annotation_value": [
-                        {"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
-                        {"name": "n_counts", "min": 3000},
-                    ],
-                    "index": [1, 99, [1000, 2000]],
-                }
-            }
-        }
-        result = self.session.put(url, json=obs_filter)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data = result.json()
-        self.assertEqual(result_data["names"], ["n_genes", "percent_mito"])
-        self.assertEqual(len(result_data["data"][0]), 3)
-        self.assertEqual(len(result_data["data"]), 15)
-
     def test_diff_exp(self):
         endpoint = "diffexp/obs"
         url = f"{URL_BASE}{endpoint}"
@@ -227,28 +148,6 @@ class EndPoints(unittest.TestCase):
         result_data = result.json()
         self.assertEqual(len(result_data), 10)
 
-    def test_get_annotations_var(self):
-        endpoint = "annotations/var"
-        url = f"{URL_BASE}{endpoint}"
-        result = self.session.get(url)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data = result.json()
-        self.assertEqual(result_data["names"], ["name", "n_cells"])
-        self.assertEqual(len(result_data["data"]), 1838)
-        self.assertEqual(len(result_data["data"][0]), 3)
-
-    def test_get_annotations_var_keys(self):
-        endpoint = "annotations/var"
-        query = "annotation-name=n_cells"
-        url = f"{URL_BASE}{endpoint}?{query}"
-        result = self.session.get(url)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data = result.json()
-        self.assertEqual(result_data["names"], ["n_cells"])
-        self.assertEqual(len(result_data["data"][0]), 2)
-
     def test_get_annotations_var_fbs(self):
         endpoint = "annotations/var"
         url = f"{URL_BASE}{endpoint}"
@@ -265,6 +164,23 @@ class EndPoints(unittest.TestCase):
         self.assertEqual(len(df['columns']), df['n_cols'])
         self.assertListEqual(df['col_idx'], ['name', 'n_cells'])
 
+    def test_get_annotations_var_keys_fbs(self):
+        endpoint = "annotations/var"
+        query = "annotation-name=n_cells"
+        url = f"{URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df['n_rows'], 1838)
+        self.assertEqual(df['n_cols'], 1)
+        self.assertIsNotNone(df['columns'])
+        self.assertIsNotNone(df['col_idx'])
+        self.assertIsNone(df['row_idx'])
+        self.assertEqual(len(df['columns']), df['n_cols'])
+        self.assertListEqual(df['col_idx'], ['n_cells'])
+
     def test_get_annotations_var_error(self):
         endpoint = "annotations/var"
         query = "annotation-name=notakey"
@@ -272,95 +188,36 @@ class EndPoints(unittest.TestCase):
         result = self.session.get(url)
         self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
 
-    def test_put_annotations_var(self):
-        endpoint = "annotations/var"
-        url = f"{URL_BASE}{endpoint}"
-        var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}]}}}
-        result = self.session.put(url, json=var_filter)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data = result.json()
-        self.assertEqual(result_data["names"], ["name", "n_cells"])
-        self.assertEqual(len(result_data["data"]), 2)
-
-    def test_filter_put_annotations_var(self):
-        endpoint = "annotations/var"
-        query = "annotation-name=n_cells"
-        url = f"{URL_BASE}{endpoint}?{query}"
-        var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}]}}}
-        result = self.session.put(url, json=var_filter)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data = result.json()
-        self.assertEqual(result_data["names"], ["n_cells"])
-        self.assertEqual(len(result_data["data"][0]), 2)
-        self.assertEqual(len(result_data["data"]), 2)
-
-    def test_get_data(self):
-        for axis in ["obs", "var"]:
-            endpoint = f"data/{axis}"
-            query = "accept-type=application/json"
-            url = f"{URL_BASE}{endpoint}?{query}"
-            result = self.session.get(url)
-            self.assertEqual(result.status_code, HTTPStatus.OK)
-            self.assertEqual(result.headers["Content-Type"], "application/json")
-            result_data = result.json()
-            self.assertEqual(len(result_data["obs"]), 2638)
-
     def test_data_mimetype_error(self):
-        for axis in ["obs", "var"]:
-            endpoint = f"data/{axis}"
-            query = "accept-type=xxx"
-            url = f"{URL_BASE}{endpoint}?{query}"
-            result = self.session.get(url)
-            self.assertEqual(result.status_code, HTTPStatus.NOT_ACCEPTABLE)
-            url = f"{URL_BASE}{endpoint}"
-            header = {"Accept": "sdkljfa;dsjalkj"}
-            result = self.session.get(url, headers=header)
-            self.assertEqual(result.status_code, HTTPStatus.NOT_ACCEPTABLE)
+        endpoint = f"data/var"
+        header = {"Accept": "xxx"}
+        url = f"{URL_BASE}{endpoint}"
+        result = self.session.put(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.NOT_ACCEPTABLE)
 
-    def test_json_default(self):
-        for axis in ["obs", "var"]:
-            endpoint = f"data/{axis}"
-            url = f"{URL_BASE}{endpoint}"
-            result = self.session.get(url)
-            self.assertEqual(result.status_code, HTTPStatus.OK)
-            self.assertEqual(result.headers["Content-Type"], "application/json")
-
-    def test_data_filter(self):
-        for axis in ["obs", "var"]:
-            endpoint = f"data/{axis}"
-            query = "accept-type=application/json&obs:louvain=NK cells&obs:louvain=CD8 T cells&obs:n_counts=3000,*"
-            url = f"{URL_BASE}{endpoint}?{query}"
-            result = self.session.get(url)
-            self.assertEqual(result.status_code, HTTPStatus.OK)
-            self.assertEqual(result.headers["Content-Type"], "application/json")
-            result_data = result.json()
-            self.assertEqual(len(result_data["obs"]), 38)
-
-    def test_data_json_put(self):
-        for axis in ["obs", "var"]:
-            endpoint = f"data/{axis}"
-            url = f"{URL_BASE}{endpoint}"
-            header = {"Accept": "application/json"}
-            obs_filter = {
-                "filter": {
-                    "obs": {
-                        "annotation_value": [
-                            {"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
-                            {"name": "n_counts", "min": 3000},
-                        ],
-                        "index": [1, 99, [1000, 2000]],
-                    }
-                }
-            }
-            result = self.session.put(url, headers=header, json=obs_filter)
-            self.assertEqual(result.status_code, HTTPStatus.OK)
-            self.assertEqual(result.headers["Content-Type"], "application/json")
-            result_data = result.json()
-            self.assertEqual(len(result_data["obs"]), 15)
+    def test_fbs_default(self):
+        endpoint = f"data/var"
+        url = f"{URL_BASE}{endpoint}"
+        result = self.session.put(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
 
     def test_data_put_fbs(self):
+        endpoint = f"data/var"
+        url = f"{URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.put(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df['n_rows'], 2638)
+        self.assertEqual(df['n_cols'], 1838)
+        self.assertIsNotNone(df['columns'])
+        self.assertListEqual(df['col_idx'].tolist(), [])
+        self.assertIsNone(df['row_idx'])
+        self.assertEqual(len(df['columns']), df['n_cols'])
+
+    def test_data_put_filter_fbs(self):
         endpoint = f"data/var"
         url = f"{URL_BASE}{endpoint}"
         header = {"Accept": "application/octet-stream"}
@@ -384,94 +241,16 @@ class EndPoints(unittest.TestCase):
         self.assertListEqual(df['col_idx'].tolist(), [0, 1, 4])
 
     def test_data_put_single_var(self):
-        for axis in ["obs", "var"]:
-            endpoint = f"data/{axis}"
-            url = f"{URL_BASE}{endpoint}"
-            header = {"Accept": "application/json"}
-            var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}}}
-            result = self.session.put(url, headers=header, json=var_filter)
-            self.assertEqual(result.status_code, HTTPStatus.OK)
-            self.assertEqual(result.headers["Content-Type"], "application/json")
-            result_data = result.json()
-            if axis == "obs":
-                self.assertEqual(len(result_data["obs"][0]), 2)
-                self.assertEqual(len(result_data["var"]), 1)
-            elif axis == "var":
-                self.assertEqual(len(result_data["obs"]), 2638)
-                self.assertEqual(len(result_data["var"][0]), 2639)
-
-    def test_cache(self):
-        endpoint = "annotations/var"
+        endpoint = f"data/var"
         url = f"{URL_BASE}{endpoint}"
-        f1 = {
-            "filter": {
-                "var": {
-                    "annotation_value": [
-                        {
-                            "name": "name",
-                            "values": [
-                                "HLA-DRB1",
-                                "HLA-DQA1",
-                                "HLA-DQB1",
-                                "HLA-DPA1",
-                                "HLA-DPB1",
-                                "MS4A1",
-                                "IL32",
-                                "CCL5",
-                                "CD79B",
-                                "CD79A",
-                            ],
-                        }
-                    ]
-                }
-            }
-        }
-        result = self.session.put(url, json=f1)
+        header = {"Accept": "application/octet-stream"}
+        var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}}}
+        result = self.session.put(url, headers=header, json=var_filter)
         self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data1 = result.json()
-        f2 = {
-            "filter": {
-                "var": {
-                    "annotation_value": [
-                        {
-                            "name": "name",
-                            "values": ["FGFBP2", "GZMA", "LTB", "PRF1", "CTSW", "GZMH", "CCL5", "CCL4", "CST7", "NKG7"],
-                        }
-                    ]
-                }
-            }
-        }
-        result = self.session.put(url, json=f2)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data2 = result.json()
-        self.assertNotEqual(result_data1, result_data2)
-
-    def test_cache_nofilter(self):
-        endpoint = "annotations/var"
-        url = f"{URL_BASE}{endpoint}"
-        f1 = {"filter": {}}
-        result = self.session.put(url, json=f1)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        result_data1 = result.json()
-        f2 = {
-            "filter": {
-                "var": {
-                    "annotation_value": [
-                        {
-                            "name": "name",
-                            "values": ["FGFBP2", "GZMA", "LTB", "PRF1", "CTSW", "GZMH", "CCL5", "CCL4", "CST7", "NKG7"],
-                        }
-                    ]
-                }
-            }
-        }
-        result = self.session.put(url, json=f2)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data2 = result.json()
-        self.assertNotEqual(result_data1, result_data2)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
 
     def test_static(self):
         endpoint = "static"

server/test/test_filter.py

@@ -1,82 +0,0 @@
-import json
-from os import path
-import unittest
-
-from numpy import float32, int32
-from werkzeug.datastructures import ImmutableMultiDict
-
-from server.app.util.filter import _convert_variable, parse_filter, QueryStringError
-
-
-class UtilTest(unittest.TestCase):
-    """Test Case for endpoints"""
-
-    def setUp(self):
-        with open(path.join(path.dirname(__file__), "schema.json")) as fh:
-            schema = json.load(fh)
-        self.schema = schema["annotations"]
-
-    def test_convert(self):
-        five = _convert_variable("int32", "5")
-        self.assertEqual(five, int32(5))
-
-    def test_convert_zero(self):
-        zero = _convert_variable("int32", "0")
-        self.assertEqual(zero, 0)
-
-    def test_convert_float(self):
-        str_to_convert = "4.38719237129"
-        val = _convert_variable("float32", str_to_convert)
-        self.assertAlmostEqual(val, float32(str_to_convert))
-
-    def test_convert_bool(self):
-        str_to_convert = "false"
-        val = _convert_variable("boolean", str_to_convert)
-        self.assertFalse(val)
-        str_to_convert = "true"
-        val = _convert_variable("boolean", str_to_convert)
-        self.assertTrue(val)
-        str_to_convert = "0"
-        with self.assertRaises(AssertionError):
-            val = _convert_variable("boolean", str_to_convert)
-
-    def test_empty_convert(self):
-        empty = _convert_variable("int32", None)
-        self.assertIsNone(empty)
-
-    def test_bad_convert(self):
-        with self.assertRaises(ValueError):
-            _convert_variable("int32", "5.5")
-
-    def test_bad_datatype(self):
-        with self.assertRaises(AssertionError):
-            _convert_variable("jkasdslkja", 1)
-
-    def test_complex_filter(self):
-        filter_dict = ImmutableMultiDict(
-            [("obs:louvain", "NK cells"), ("obs:louvain", "CD8 T cells"), ("obs:n_counts", "3000,*")]
-        )
-        filter_ = parse_filter(filter_dict, self.schema)
-        self.assertIn("obs", filter_)
-        self.assertEqual(
-            filter_["obs"]["annotation_value"],
-            [
-                {"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
-                {"name": "n_counts", "max": None, "min": 3000.0},
-            ],
-        )
-
-    def test_bad_filter(self):
-        bad_annotation_type = ImmutableMultiDict([("obs:tissue", "lung")])
-        with self.assertRaises(QueryStringError):
-            parse_filter(bad_annotation_type, self.schema)
-        bad_axis = ImmutableMultiDict([("xyz:n_genes", "100,1000")])
-        with self.assertRaises(QueryStringError):
-            parse_filter(bad_axis, self.schema)
-
-    def test_boolean_filter(self):
-        schema = {"obs": [{"name": "bool_filter", "type": "boolean"}]}
-        filter_dict = ImmutableMultiDict([("obs:bool_filter", "false")])
-        filter_ = parse_filter(filter_dict, schema)
-        self.assertIn("obs", filter_)
-        self.assertEqual(filter_["obs"]["annotation_value"], [{"name": "bool_filter", "values": [False]}])

server/test/test_nan_rest.py

@@ -2,6 +2,9 @@ from http import HTTPStatus
 from subprocess import Popen
 import unittest
 import time
+import math
+
+import decode_fbs
 
 import requests
 
@@ -43,9 +46,29 @@ class WithNaNs(unittest.TestCase):
         result = self.session.get(url)
         self.assertEqual(result.status_code, HTTPStatus.OK)
 
-    def test_errors(self):
-        endpoints = ["annotations/obs", "annotations/var", "data/obs", "data/var"]
-        for endpoint in endpoints:
-            url = f"{URL_BASE}{endpoint}"
-            result = self.session.get(url)
-            self.assertEqual(result.status_code, HTTPStatus.INTERNAL_SERVER_ERROR)
+    def test_data(self):
+        endpoint = "data/var"
+        url = f"{URL_BASE}{endpoint}"
+        result = self.session.put(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertTrue(math.isnan(df["columns"][3][3]))
+
+    def test_annotation_obs(self):
+        endpoint = "annotations/obs"
+        url = f"{URL_BASE}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertTrue(math.isnan(df["columns"][2][0]))
+
+    def test_annotation_var(self):
+        endpoint = "annotations/var"
+        url = f"{URL_BASE}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertTrue(math.isnan(df["columns"][2][0]))

server/test/test_nan_scanpy_engine.py

@@ -1,4 +1,3 @@
-import json
 import pytest
 import unittest
 import warnings
@@ -7,7 +6,7 @@ import math
 import decode_fbs
 
 from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
-from server.app.util.errors import JSONEncodingValueError
+from server.app.util.errors import FilterError
 
 
 class NaNTest(unittest.TestCase):
@@ -42,10 +41,15 @@ class NaNTest(unittest.TestCase):
         self.assertEqual(data_frame_var["n_cols"], 100)
         self.assertTrue(math.isnan(data_frame_var["columns"][3][3]))
 
-        with pytest.raises(JSONEncodingValueError):
-            json.loads(self.data.data_frame(None, "obs"))
-        with pytest.raises(JSONEncodingValueError):
-            json.loads(self.data.data_frame(None, "var"))
+        with pytest.raises(FilterError):
+            self.data.data_frame_to_fbs_matrix("an erroneous filter", "var")
+        with pytest.raises(FilterError):
+            filter_ = {
+                "filter": {
+                    "obs": {"index": [1, 99, [200, 300]]}
+                }
+            }
+            self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
 
     def test_dataframe_obs_not_implemented(self):
         with self.assertRaises(ValueError) as cm:
@@ -65,8 +69,3 @@ class NaNTest(unittest.TestCase):
         self.assertEqual(annotations["col_idx"], ["name", "n_cells", "var_with_nans"])
         self.assertEqual(annotations["n_rows"], 100)
         self.assertTrue(math.isnan(annotations["columns"][2][0]))
-
-        with pytest.raises(JSONEncodingValueError):
-            json.loads(self.data.annotation(None, "obs"))
-        with pytest.raises(JSONEncodingValueError):
-            json.loads(self.data.annotation(None, "var"))

server/test/test_scanpy_engine.py

@@ -3,11 +3,13 @@ from os import path
 import pytest
 import time
 import unittest
+import decode_fbs
 
 import numpy as np
 from pandas import Series
 
 from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
+from server.app.util.errors import FilterError
 
 
 class UtilTest(unittest.TestCase):
@@ -45,55 +47,29 @@ class UtilTest(unittest.TestCase):
     def test_filter_idx(self):
         filter_ = {
             "filter": {
-                "var": {"index": [1, 99, [200, 300]]},
-                "obs": {"index": [1, 99, [1000, 2000]]},
+                "var": {"index": [1, 99, [200, 300]]}
             }
         }
-        data = self.data.filter_dataframe(filter_["filter"])
-        self.assertEqual(data.shape, (1002, 102))
-
-    def test_filter_annotation(self):
-        filter_ = {
-            "filter": {
-                "obs": {
-                    "annotation_value": [
-                        {"name": "louvain", "values": ["NK cells", "CD8 T cells"]}
-                    ]
-                }
-            }
-        }
-        data = self.data.filter_dataframe(filter_["filter"])
-        self.assertEqual(data.shape, (470, 1838))
-        filter_ = {
-            "filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}
-        }
-        data = self.data.filter_dataframe(filter_["filter"])
-        self.assertEqual(data.shape, (497, 1838))
-
-    def test_filter_annotation_no_uns(self):
-        filter_ = {
-            "filter": {
-                "var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}
-            }
-        }
-        data = self.data.filter_dataframe(filter_["filter"])
-        self.assertEqual(data.shape[1], 1)
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 102)
 
     def test_filter_complex(self):
         filter_ = {
             "filter": {
-                "var": {"index": [1, 99, [200, 300]]},
-                "obs": {
+                "var": {
                     "annotation_value": [
-                        {"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
-                        {"name": "n_counts", "min": 3000},
+                        {"name": "n_cells", "min": 10}
                     ],
-                    "index": [1, 99, [1000, 2000]],
-                },
+                    "index": [1, 99, [200, 300]]
+                }
             }
         }
-        data = self.data.filter_dataframe(filter_["filter"])
-        self.assertEqual(data.shape, (15, 102))
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 91)
 
     def test_obs_and_var_names(self):
         self.assertEqual(np.sum(self.data.data.var["name"].isna()), 0)
@@ -119,60 +95,42 @@ class UtilTest(unittest.TestCase):
         )
 
     def test_layout(self):
-        layout = json.loads(self.data.layout(None))
-        self.assertEqual(layout["layout"]["ndims"], 2)
-        self.assertEqual(len(layout["layout"]["coordinates"]), 2638)
-        self.assertEqual(layout["layout"]["coordinates"][0][0], 0)
-        for idx, val in enumerate(layout["layout"]["coordinates"]):
-            self.assertLessEqual(val[1], 1)
-            self.assertLessEqual(val[2], 1)
+        fbs = self.data.layout_to_fbs_matrix()
+        layout = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(layout["n_cols"], 2)
+        self.assertEqual(layout["n_rows"], 2638)
+
+        X = layout["columns"][0]
+        self.assertTrue((X >= 0).all() and (X <= 1).all())
+        Y = layout["columns"][1]
+        self.assertTrue((Y >= 0).all() and (Y <= 1).all())
 
     def test_annotations(self):
-        annotations = json.loads(self.data.annotation(None, "obs"))
+        fbs = self.data.annotation_to_fbs_matrix("obs")
+        annotations = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(annotations["n_rows"], 2638)
+        self.assertEqual(annotations["n_cols"], 5)
         self.assertEqual(
-            annotations["names"],
+            annotations["col_idx"],
             ["name", "n_genes", "percent_mito", "n_counts", "louvain"],
         )
-        self.assertEqual(len(annotations["data"]), 2638)
-        annotations = json.loads(self.data.annotation(None, "var"))
-        self.assertEqual(annotations["names"], ["name", "n_cells"])
-        self.assertEqual(len(annotations["data"]), 1838)
+
+        fbs = self.data.annotation_to_fbs_matrix("var")
+        annotations = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(annotations['n_rows'], 1838)
+        self.assertEqual(annotations['n_cols'], 2)
+        self.assertEqual(annotations["col_idx"], ["name", "n_cells"])
 
     def test_annotation_fields(self):
-        annotations = json.loads(
-            self.data.annotation(None, "obs", ["n_genes", "n_counts"])
-        )
-        self.assertEqual(annotations["names"], ["n_genes", "n_counts"])
-        self.assertEqual(len(annotations["data"]), 2638)
-        annotations = json.loads(self.data.annotation(None, "var", ["name"]))
-        self.assertEqual(annotations["names"], ["name"])
-        self.assertEqual(len(annotations["data"]), 1838)
+        fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"])
+        annotations = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(annotations["n_rows"], 2638)
+        self.assertEqual(annotations['n_cols'], 2)
 
-    def test_filtered_annotation(self):
-        filter_ = {
-            "filter": {
-                "obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]},
-                "var": {
-                    "annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}]
-                },
-            }
-        }
-        annotations = json.loads(self.data.annotation(filter_["filter"], "obs"))
-        self.assertEqual(
-            annotations["names"],
-            ["name", "n_genes", "percent_mito", "n_counts", "louvain"],
-        )
-        self.assertEqual(len(annotations["data"]), 497)
-        annotations = json.loads(self.data.annotation(filter_["filter"], "var"))
-        self.assertEqual(annotations["names"], ["name", "n_cells"])
-        self.assertEqual(len(annotations["data"]), 2)
-
-    def test_filtered_layout(self):
-        filter_ = {
-            "filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}
-        }
-        layout = json.loads(self.data.layout(filter_["filter"]))
-        self.assertEqual(len(layout["layout"]["coordinates"]), 497)
+        fbs = self.data.annotation_to_fbs_matrix("var", ["name"])
+        annotations = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(annotations['n_rows'], 1838)
+        self.assertEqual(annotations['n_cols'], 1)
 
     def test_diffexp_topN(self):
         f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
@@ -183,42 +141,51 @@ class UtilTest(unittest.TestCase):
         self.assertEqual(len(result), 20)
 
     def test_data_frame(self):
-        data_frame_obs = json.loads(self.data.data_frame(None, "obs"))
-        self.assertEqual(len(data_frame_obs["var"]), 1838)
-        self.assertEqual(len(data_frame_obs["obs"]), 2638)
-        data_frame_var = json.loads(self.data.data_frame(None, "var"))
-        self.assertEqual(len(data_frame_var["var"]), 1838)
-        self.assertEqual(len(data_frame_var["obs"]), 2638)
+        fbs = self.data.data_frame_to_fbs_matrix(None, "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 1838)
+
+        with self.assertRaises(ValueError):
+            self.data.data_frame_to_fbs_matrix(None, "obs")
 
     def test_filtered_data_frame(self):
+        filter_ = {
+            "filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 100}]}}
+        }
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 1040)
+
         filter_ = {
             "filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}
         }
-        data_frame_obs = json.loads(self.data.data_frame(filter_["filter"], "obs"))
-        self.assertEqual(len(data_frame_obs["var"]), 1838)
-        self.assertEqual(len(data_frame_obs["obs"]), 497)
-        self.assertIsInstance(data_frame_obs["obs"][0], (list, tuple))
-        self.assertEqual(type(data_frame_obs["var"][0]), int)
-        data_frame_var = json.loads(self.data.data_frame(filter_["filter"], "var"))
-        self.assertEqual(len(data_frame_var["var"]), 1838)
-        self.assertEqual(len(data_frame_var["obs"]), 497)
-        self.assertIsInstance(data_frame_var["var"][0], (list, tuple))
-        self.assertEqual(type(data_frame_var["obs"][0]), int)
+        with self.assertRaises(FilterError):
+            self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
 
-    def test_data_single_gene(self):
-        for axis in ["obs", "var"]:
-            filter_ = {
-                "filter": {
-                    "var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}
-                }
+    def test_data_named_gene(self):
+        filter_ = {
+            "filter": {
+                "var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}
             }
-            data_frame_var = json.loads(self.data.data_frame(filter_["filter"], axis))
-            if axis == "obs":
-                self.assertEqual(type(data_frame_var["var"][0]), int)
-                self.assertIsInstance(data_frame_var["obs"][0], (list, tuple))
-            elif axis == "var":
-                self.assertEqual(type(data_frame_var["obs"][0]), int)
-                self.assertIsInstance(data_frame_var["var"][0], (list, tuple))
+        }
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 1)
+        self.assertEqual(data["col_idx"], [4])
+
+        filter_ = {
+            "filter": {
+                "var": {"annotation_value": [{"name": "name", "values": ["SPEN", "TYMP", "PRMT2"]}]}
+            }
+        }
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 3)
+        self.assertTrue((data["col_idx"] == [15, 1818, 1837]).all())
 
     if __name__ == "__main__":
         unittest.main()

setup.py

@@ -17,7 +17,7 @@ with open("server/requirements.txt") as fh:
 
 setup(
     name="cellxgene",
-    version="0.6.1",
+    version="0.7.0",
     packages=find_packages(),
     url="https://github.com/chanzuckerberg/cellxgene",
     license="MIT",