bump version (#684)

* add continuousSelection reducer and make histo respond to state changes

* correctly handle uninitialized state for brush move

* PR review tweaks

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.6.1
+current_version = 0.8.0
 
 [bumpversion:file:setup.py]
 search = version="{current_version}"

.travis.yml

@@ -1,8 +1,8 @@
 language: python
-python:
-  - "3.6"
+dist: xenial
+sudo: required
 node_js:
-  - "8"
+  - 8
 cache:
   pip: true
 install:
@@ -12,10 +12,17 @@ install:
   - make install
   - pip install -r server/requirements-dev.txt
   - docker build .
-script:
-  - set -eo pipefail
-  - flake8 server
-  - black --check
-  - npm run --prefix client/ build
-  - npm run --prefix client/ test
-  - pytest -s server/test
+
+jobs:
+  include:
+  - name: "Branch Tests 3.7"
+    python: "3.7"
+    script: ./travis-build.sh
+  - name: "Branch Tests 3.6"
+    python: "3.6"
+    script: ./travis-build.sh
+  - name: "Smoke Tests"
+    python: "3.6"
+    if: branch = master AND type = cron
+    script:
+      - npm run --prefix client/ smoke-test

README.md

@@ -37,7 +37,7 @@ You should see your web browser open with the following
 
 There are several options available, such as:
 
-- `--layout` to specify the layout as `tsne` or `umap`
+- `--layout` to specify the layout as `tsne`, `umap`, `diffmap`, `phate`, `draw_graph_fa`, or `draw_graph_fr`
 - `--title` to show a title on the explorer
 - `--open` to automatically open the web browser after launching (OS X only)
 
@@ -57,7 +57,7 @@ The `launch` command assumes that the data is stored in the `.h5ad` format from
 
 - an `obs` field has a unique identifier for every cell (you can specify which field to use with the `--obs-names` option, by default it will use the value of `data.obs_names`)
 - a `var` field has a unique identifier for every gene (you can specify which field to use with the `--var-names` option, by default it will use the value of `data.var_names`)
-- an `obsm` field contains the two-dimensional coordinates for the layout that you want to render (e.g. `X_tsne` for the `tsne` layout or `X_umap` for the `umap` layout)
+- an `obsm` field contains the two-dimensional coordinates for the layout that you want to render (e.g. `X_umap` for the `umap` layout)
 - any additional `obs` fields will be rendered as per-cell continuous or categorical metadata by the app (e.g. `louvain` cluster assignments)
 
 ### prepare
@@ -240,10 +240,10 @@ Then clone the project
 git clone https://github.com/chanzuckerberg/cellxgene.git
 ```
 
-Build the client web assets by calling this from inside the `cellxgene` folder
+Build the client web assets by calling `make` from inside the `cellxgene` folder
 
 ```
-./bin/build-client
+make
 ```
 
 Install all requirements (we recommend doing this inside a virtual environment)

client/__tests__/e2e/config.js

@@ -0,0 +1,6 @@
+export const jest_env = process.env.JEST_ENV || "dev";
+export const appPort = process.env.JEST_CXG_PORT || 3000;
+export const appUrlBase = `http://localhost:${appPort}`;
+export const DEV = jest_env === "dev";
+export const DEBUG = jest_env === "debug";
+export const DATASET = "pbmc3k";

client/__tests__/e2e/data.js

@@ -0,0 +1,105 @@
+export const datasets = {
+  pbmc3k: {
+    title: "cellxgene: pbmc3k",
+    dataframe: {
+      nObs: "2638",
+      nVar: "1838",
+      type: "float32"
+    },
+    categorical: {
+      louvain: {
+        "B cells": "342",
+        "CD14+ Monocytes": "480",
+        "CD4 T cells": "1144",
+        "CD8 T cells": "316",
+        "Dendritic cells": "37",
+        "FCGR3A+ Monocytes": "150",
+        Megakaryocytes: "15",
+        "NK cells": "154"
+      }
+    },
+    continuous: {
+      n_genes: "int32",
+      percent_mito: "float32",
+      n_counts: "float32"
+    },
+    cellsets: {
+      lasso: [
+        {
+          "coordinates-as-percent": { x1: 0.25, y1: 0.25, x2: 0.35, y2: 0.35 },
+          count: "26"
+        }
+      ],
+      categorical: [
+        {
+          metadata: "louvain",
+          values: ["B cells", "Megakaryocytes"],
+          count: "357"
+        }
+      ],
+      continuous: [
+        {
+          metadata: "n_genes",
+          "coordinates-as-percent": { x1: 0.25, y1: 0.5, x2: 0.55, y2: 0.5 },
+          count: "1537"
+        }
+      ]
+    },
+
+    diffexp: {
+      cellset1: [
+        { kind: "categorical", metadata: "louvain", values: ["B cells"] }
+      ],
+      cellset2: [
+        {
+          kind: "categorical",
+          metadata: "louvain",
+          values: ["CD4 T cells", "NK cells"]
+        }
+      ],
+      "gene-results": [
+        "HLA-DRB1",
+        "HLA-DPB1",
+        "CD79A",
+        "HLA-DPA1",
+        "HLA-DQA1",
+        "CD79B",
+        "HLA-DQB1",
+        "MS4A1",
+        "IL32",
+        "CD37"
+      ]
+    },
+
+    genes: {
+      bulkadd: ["S100A8", "FCGR3A", "LGALS2", "GSTP1"],
+      search: "ACD"
+    },
+    subset: {
+      cellset1: [
+        {
+          kind: "categorical",
+          metadata: "louvain",
+          values: ["B cells", "Megakaryocytes"]
+        }
+      ],
+      count: "357",
+      categorical: {
+        louvain: {
+          "B cells": "342",
+          Megakaryocytes: "15"
+        }
+      },
+      lasso: {
+        "coordinates-as-percent": { x1: 0.45, y1: 0.45, x2: 0.5, y2: 0.5 },
+        count: "67"
+      }
+    },
+    scatter: {
+      genes: { x: "S100A8", y: "FCGR3A" }
+    },
+    pan: {
+      "coordinates-as-percent": { x1: 0.75, y1: 0.75, x2: 0.35, y2: 0.35 }
+    }
+  }
+};

client/__tests__/e2e/e2e.test.js

@@ -0,0 +1,284 @@
+import puppeteer from "puppeteer";
+import { appUrlBase, DEBUG, DEV, DATASET } from "./config";
+import { puppeteerUtils, cellxgeneActions } from "./puppeteerUtils";
+import { datasets } from "./data";
+
+let browser, page, utils, cxgActions, spy;
+const browserViewport = { width: 1280, height: 960 };
+let data = datasets[DATASET];
+
+if (DEBUG) jest.setTimeout(100000);
+if (DEV) jest.setTimeout(10000);
+
+beforeAll(async () => {
+  const browserParams = DEV
+    ? { headless: false, slowMo: 5 }
+    : DEBUG
+    ? { headless: false, slowMo: 100, devtools: true }
+    : {};
+  browser = await puppeteer.launch(browserParams);
+  page = await browser.newPage();
+  await page.setViewport(browserViewport);
+  if (DEV || DEBUG) {
+    page.on("console", msg => console.log(`PAGE LOG: ${msg.text()}`));
+  }
+  page.on("pageerror", err => {
+    throw new Error(`Console error: ${err}`);
+  });
+  utils = puppeteerUtils(page);
+  cxgActions = cellxgeneActions(page);
+});
+
+beforeEach(async () => {
+  await page.goto(appUrlBase);
+});
+
+afterAll(() => {
+  if (!DEBUG) {
+    browser.close();
+  }
+});
+
+describe("did launch", async () => {
+  test("page launched", async () => {
+    let el = await utils.getOneElementInnerHTML("[data-testid='header']");
+    expect(el).toBe(data.title);
+  });
+});
+
+describe("metadata loads", async () => {
+  test("categories and values from dataset appear", async () => {
+    for (const label in data.categorical) {
+      await utils.waitByID(`category-${label}`);
+      const categoryName = await utils.getOneElementInnerText(
+        `[data-testid="category-${label}"]`
+      );
+      expect(categoryName).toMatch(label);
+      await utils.clickOn(`category-expand-${label}`);
+      const categories = await cxgActions.getAllCategoriesAndCounts(label);
+      expect(Object.keys(categories)).toMatchObject(
+        Object.keys(data.categorical[label])
+      );
+      expect(Object.values(categories)).toMatchObject(
+        Object.values(data.categorical[label])
+      );
+    }
+  });
+
+  test("continuous data appears", async () => {
+    for (const label in data.continuous) {
+      await utils.waitByID(`histogram-${label}`);
+    }
+  });
+});
+
+describe("cell selection", async () => {
+  test("selects all cells cellset 1", async () => {
+    const cellCount = await cxgActions.cellSet(1);
+    expect(cellCount).toBe(data.dataframe.nObs);
+  });
+
+  test("selects all cells cellset 2", async () => {
+    const cellCount = await cxgActions.cellSet(2);
+    expect(cellCount).toBe(data.dataframe.nObs);
+  });
+
+  test("selects cells via lasso", async () => {
+    for (const cellset of data.cellsets.lasso) {
+      const cellset1 = await cxgActions.calcDragCoordinates(
+        "layout-graph",
+        cellset["coordinates-as-percent"]
+      );
+      await cxgActions.drag("layout-graph", cellset1.start, cellset1.end, true);
+      const cellCount = await cxgActions.cellSet(1);
+      expect(cellCount).toBe(cellset.count);
+    }
+  });
+
+  test("selects cells via categorical", async () => {
+    for (const cellset of data.cellsets.categorical) {
+      await utils.clickOn(`category-expand-${cellset.metadata}`);
+      await utils.clickOn(`category-select-${cellset.metadata}`);
+      for (const val of cellset.values) {
+        await utils.clickOn(
+          `categorical-value-select-${cellset.metadata}-${val}`
+        );
+      }
+      const cellCount = await cxgActions.cellSet(1);
+      expect(cellCount).toBe(cellset.count);
+    }
+  });
+
+  test("selects cells via continuous", async () => {
+    for (const cellset of data.cellsets.continuous) {
+      const histId = `histogram-${cellset.metadata}-plot-brush`;
+      const coords = await cxgActions.calcDragCoordinates(
+        histId,
+        cellset["coordinates-as-percent"]
+      );
+      await cxgActions.drag(histId, coords.start, coords.end);
+      const cellCount = await cxgActions.cellSet(1);
+      expect(cellCount).toBe(cellset.count);
+    }
+  });
+});
+
+describe("gene entry", async () => {
+  test("search for single gene", async () => {
+    // blueprint's  typeahead is treating typing weird, clicking & waiting first solves this
+    await utils.typeInto("gene-search", data.genes.search);
+    await page.keyboard.press("Enter");
+    await page.waitForSelector(
+      `[data-testid='histogram-${data.genes.search}']`
+    );
+  });
+
+  test("bulk add genes", async () => {
+    await cxgActions.reset();
+    const testGenes = data.genes.bulkadd;
+    await utils.clickOn("section-bulk-add");
+    await utils.typeInto("input-bulk-add", testGenes.join(","));
+    await page.keyboard.press("Enter");
+    const userGeneHist = await cxgActions.getAllHistograms(
+      "histogram-user-gene"
+    );
+    expect(userGeneHist).toEqual(expect.arrayContaining(testGenes));
+  });
+});
+
+describe("diffexp", async () => {
+  test("selects cells, saves them and performs diffexp", async () => {
+    for (const select of data.diffexp.cellset1) {
+      if (select.kind === "categorical") {
+        await cxgActions.selectCategory(select.metadata, select.values, true);
+      }
+    }
+    await cxgActions.cellSet(1);
+    for (const select of data.diffexp.cellset2) {
+      if (select.kind === "categorical") {
+        await cxgActions.selectCategory(select.metadata, select.values, true);
+      }
+    }
+    await cxgActions.cellSet(2);
+    await utils.clickOn("diffexp-button");
+    const diffExpHists = await cxgActions.getAllHistograms("histogram-diffexp");
+    expect(diffExpHists).toEqual(
+      expect.arrayContaining(data.diffexp["gene-results"])
+    );
+  });
+});
+//
+
+describe("subset/reset", async () => {
+  test("subset - cell count matches", async () => {
+    for (const select of data.subset.cellset1) {
+      if (select.kind === "categorical") {
+        await cxgActions.selectCategory(select.metadata, select.values, true);
+      }
+    }
+    await utils.clickOn("subset-button");
+    for (const label in data.subset.categorical) {
+      const categories = await cxgActions.getAllCategoriesAndCounts(label);
+      expect(Object.keys(categories)).toMatchObject(
+        Object.keys(data.subset.categorical[label])
+      );
+      expect(Object.values(categories)).toMatchObject(
+        Object.values(data.subset.categorical[label])
+      );
+    }
+  });
+
+  test("reset after subset", async () => {
+    for (const select of data.subset.cellset1) {
+      if (select.kind === "categorical") {
+        await cxgActions.selectCategory(select.metadata, select.values, true);
+      }
+    }
+    await utils.clickOn("subset-button");
+    for (const label in data.subset.categorical) {
+      const categories = await cxgActions.getAllCategoriesAndCounts(label);
+      expect(Object.keys(categories)).toMatchObject(
+        Object.keys(data.subset.categorical[label])
+      );
+      expect(Object.values(categories)).toMatchObject(
+        Object.values(data.subset.categorical[label])
+      );
+    }
+    await cxgActions.reset();
+    for (const label in data.categorical) {
+      await utils.waitByID(`category-${label}`);
+      const categoryName = await utils.getOneElementInnerText(
+        `[data-testid="category-${label}"]`
+      );
+      expect(categoryName).toMatch(label);
+      const categories = await cxgActions.getAllCategoriesAndCounts(label);
+      expect(Object.keys(categories)).toMatchObject(
+        Object.keys(data.categorical[label])
+      );
+      expect(Object.values(categories)).toMatchObject(
+        Object.values(data.categorical[label])
+      );
+    }
+  });
+
+  test("lasso after subset", async () => {
+    for (const select of data.subset.cellset1) {
+      if (select.kind === "categorical") {
+        await cxgActions.selectCategory(select.metadata, select.values, true);
+      }
+    }
+    await utils.clickOn("subset-button");
+    const lassoSelection = await cxgActions.calcDragCoordinates(
+      "layout-graph",
+      data.subset.lasso["coordinates-as-percent"]
+    );
+    await cxgActions.drag(
+      "layout-graph",
+      lassoSelection.start,
+      lassoSelection.end,
+      true
+    );
+    const cellCount = await cxgActions.cellSet(1);
+    expect(cellCount).toBe(data.subset.lasso.count);
+  });
+});
+
+describe("scatter plot", async () => {
+  test("scatter plot appears", async () => {
+    await cxgActions.reset();
+    const testGenes = data.scatter.genes;
+    await utils.clickOn("section-bulk-add");
+    await utils.typeInto("input-bulk-add", Object.values(testGenes).join(","));
+    await page.keyboard.press("Enter");
+    await utils.clickOn(`plot-x-${data.scatter.genes.x}`);
+    await utils.clickOn(`plot-y-${data.scatter.genes.y}`);
+    await utils.waitByID("scatterplot");
+  });
+});
+
+// interact with UI elements just that they do not break
+describe("ui elements don't error", async () => {
+  test("color by", async () => {
+    for (const label in data.categorical) {
+      await utils.clickOn(`colorby-${label}`);
+    }
+    for (const label in data.continuous) {
+      await utils.clickOn(`colorby-${label}`);
+    }
+  });
+
+  test("pan and zoom", async () => {
+    await utils.clickOn("mode-pan-zoom");
+    const panCoords = await cxgActions.calcDragCoordinates(
+      "layout-graph",
+      data.pan["coordinates-as-percent"]
+    );
+    await cxgActions.drag(
+      "layout-graph",
+      panCoords.start,
+      panCoords.end,
+      false
+    );
+    await page.evaluate(`window.scrollBy(0, 1000);`);
+  });
+});

client/__tests__/e2e/e2eJestConfig.json

@@ -0,0 +1,10 @@
+{
+  "preset": "jest-puppeteer",
+  "testMatch": [
+    "**/__tests__/**/?(*.)(spec|test).js?(x)"
+  ],
+  "testURL": "http://localhost/",
+  "setupFiles": [
+    "../setupMissingGlobals.js"
+  ]
+}

client/__tests__/e2e/puppeteerUtils.js

@@ -0,0 +1,161 @@
+export const puppeteerUtils = puppeteerPage => ({
+  async waitByID(testid) {
+    return await puppeteerPage.waitForSelector(`[data-testid='${testid}']`);
+  },
+
+  async waitByClass(testclass) {
+    return await puppeteerPage.waitForSelector(
+      `[data-testclass='${testclass}']`
+    );
+  },
+
+  async typeInto(testid, text) {
+    // only works for text without special characters
+    await this.waitByID(testid);
+    // type ahead can be annoying if you don't pause before you type
+    await puppeteerPage.click(`[data-testid='${testid}']`);
+    await puppeteerPage.waitFor(200);
+    await puppeteerPage.type(`[data-testid='${testid}']`, text);
+  },
+
+  async clickOn(testid) {
+    await this.waitByID(testid);
+    await puppeteerPage.click(`[data-testid='${testid}']`);
+    await puppeteerPage.waitFor(50);
+  },
+
+  async getOneElementInnerHTML(selector) {
+    let text = await puppeteerPage.$eval(selector, el => el.innerHTML);
+    return text;
+  },
+
+  async getOneElementInnerText(selector) {
+    let text = await puppeteerPage.$eval(selector, el => el.innerText);
+    return text;
+  }
+});
+
+export const cellxgeneActions = puppeteerPage => ({
+  async drag(testid, start, end, lasso = false) {
+    const layout = await puppeteerUtils(puppeteerPage).waitByID(testid);
+    const elBox = await layout.boxModel();
+    const x1 = elBox.content[0].x + start.x;
+    const x2 = elBox.content[0].x + end.x;
+    const y1 = elBox.content[0].y + start.y;
+    const y2 = elBox.content[0].y + end.y;
+    await puppeteerPage.mouse.move(x1, y1);
+    await puppeteerPage.mouse.down();
+    if (lasso) {
+      await puppeteerPage.mouse.move(x2, y1);
+      await puppeteerPage.mouse.move(x2, y2);
+      await puppeteerPage.mouse.move(x1, y2);
+      await puppeteerPage.mouse.move(x1, y1);
+    } else {
+      await puppeteerPage.mouse.move(x2, y2);
+    }
+    await puppeteerPage.mouse.up();
+  },
+
+  async getAllHistograms(testclass) {
+    await puppeteerUtils(puppeteerPage).waitByClass(testclass);
+    const histograms = await puppeteerPage.$$eval(
+      `[data-testclass=${testclass}]`,
+      els => {
+        return els.map(el => {
+          return el.dataset.testid.substring(
+            "histogram_".length,
+            el.dataset.testid.length
+          );
+        });
+      }
+    );
+    return histograms;
+  },
+
+  async getAllCategoriesAndCounts(category) {
+    await puppeteerUtils(puppeteerPage).waitByClass("categorical-row");
+    const categories = await puppeteerPage.$$eval(
+      `[data-testid="category-${category}"] [data-testclass='categorical-row']`,
+      els => {
+        let result = {};
+        els.forEach(el => {
+          const cat = el.querySelector("[data-testclass='categorical-value']")
+            .innerText;
+          const count = el.querySelector(
+            "[data-testclass='categorical-value-count']"
+          ).innerText;
+          result[cat] = count;
+        });
+        return result;
+      }
+    );
+    return categories;
+  },
+
+  async cellSet(num) {
+    await puppeteerUtils(puppeteerPage).clickOn(`cellset-button-${num}`);
+    return await puppeteerUtils(puppeteerPage).getOneElementInnerText(
+      `[data-testid='cellset-count-${num}']`
+    );
+  },
+
+  async resetCategory(category) {
+    const checkboxId = `category-select-${category}`;
+    await puppeteerUtils(puppeteerPage).waitByID(checkboxId);
+    const checkedPseudoclass = await puppeteerPage.$eval(
+      `[data-testid='${checkboxId}']`,
+      el => {
+        return el.matches(":checked");
+      }
+    );
+    if (!checkedPseudoclass) {
+      await puppeteerUtils(puppeteerPage).clickOn(checkboxId);
+    }
+    try {
+      const categoryRow = await puppeteerUtils(puppeteerPage).waitByID(
+        `category-expand-${category}`
+      );
+      const isExpanded = await categoryRow.$(
+        "[data-testclass='category-expand-is-expanded']"
+      );
+      if (isExpanded) {
+        await puppeteerUtils(puppeteerPage).clickOn(
+          `category-expand-${category}`
+        );
+      }
+    } catch {}
+  },
+
+  async calcDragCoordinates(testid, coordinateAsPercent) {
+    const el = await puppeteerUtils(puppeteerPage).waitByID(testid);
+    const size = await el.boxModel();
+    const coords = {
+      start: {
+        x: Math.floor(size.width * coordinateAsPercent.x1),
+        y: Math.floor(size.height * coordinateAsPercent.y1)
+      },
+      end: {
+        x: Math.floor(size.width * coordinateAsPercent.x2),
+        y: Math.floor(size.height * coordinateAsPercent.y2)
+      }
+    };
+    return coords;
+  },
+
+  async selectCategory(category, values, reset = true) {
+    if (reset) await this.resetCategory(category);
+    await puppeteerUtils(puppeteerPage).clickOn(`category-expand-${category}`);
+    await puppeteerUtils(puppeteerPage).clickOn(`category-select-${category}`);
+    for (const val of values) {
+      await puppeteerUtils(puppeteerPage).clickOn(
+        `categorical-value-select-${category}-${val}`
+      );
+    }
+  },
+
+  async reset() {
+    await puppeteerUtils(puppeteerPage).clickOn("reset");
+    // loading state never actually happens, reset is too fast
+    await page.waitFor(200);
+  }
+});

client/__tests__/reducers/cascade.test.js

@@ -0,0 +1,48 @@
+import cascadeReducers from "../../src/reducers/cascade";
+
+describe("create", () => {
+  test("from Array", () => {
+    expect(cascadeReducers([["foo", () => 0]])).toBeInstanceOf(Function);
+  });
+
+  test("from Map", () => {
+    expect(cascadeReducers(new Map([["foo", () => 0]]))).toBeInstanceOf(
+      Function
+    );
+  });
+});
+
+describe("cascade", () => {
+  test("expected arguments provided & cascade ordering", () => {
+    const topLevelState = {};
+    const topLevelAction = { type: "test" };
+
+    const reducer = cascadeReducers([
+      [
+        "foo",
+        (currentState, action, nextSharedState, prevSharedState) => {
+          expect(currentState).toBeUndefined();
+          expect(action).toEqual(topLevelAction);
+          expect(nextSharedState).toStrictEqual({});
+          expect(prevSharedState).toBe(topLevelState);
+          return 0;
+        }
+      ],
+      [
+        "bar",
+        (currentState, action, nextSharedState, prevSharedState) => {
+          expect(currentState).toBeUndefined();
+          expect(action).toEqual(topLevelAction);
+          expect(nextSharedState).toStrictEqual({ foo: 0 });
+          expect(prevSharedState).toBe(topLevelState);
+          return 99;
+        }
+      ]
+    ]);
+
+    const nextState = reducer(topLevelState, topLevelAction);
+    expect(nextState).toStrictEqual({ foo: 0, bar: 99 });
+    expect(topLevelState).toStrictEqual({});
+    expect(topLevelAction).toStrictEqual({ type: "test" });
+  });
+});

client/__tests__/reducers/undoable.test.js

@@ -0,0 +1,87 @@
+import undoable from "../../src/reducers/undoable";
+
+describe("create", () => {
+  test("no keys", () => {
+    expect(() => undoable(() => {})).toThrow();
+    expect(() => undoable(() => {}, null)).toThrow();
+    expect(() => undoable(() => {}, [])).toThrow();
+    expect(() => undoable(() => {}, [], {})).toThrow();
+  });
+
+  test("simple", () => {
+    expect(undoable(() => {}, ["foo"])).toBeInstanceOf(Function);
+    expect(undoable(() => {}, ["foo"], {})).toBeInstanceOf(Function);
+  });
+
+  test("handles undefined initial state", () => {
+    expect(
+      undoable(() => {}, ["a"])(undefined, { type: "test" })
+    ).toMatchObject({});
+  });
+});
+
+describe("undo", () => {
+  test("expected state modifications", () => {
+    const initialState = { a: 0, b: 1000 };
+    const reducer = state => {
+      return { a: state.a + 1, b: state.b + 1 };
+    };
+    const undoableReducer = undoable(reducer, ["a"]);
+
+    const s1 = undoableReducer(initialState, { type: "test" });
+    expect(s1).toMatchObject({ a: 1, b: 1001 });
+
+    // test that only specified keys are undone
+    const s2 = undoableReducer(s1, { type: "@@undoable/undo" });
+    expect(s2).toMatchObject({ a: 0, b: 1001 });
+
+    // test backstop when no more history
+    const s3 = undoableReducer(s2, { type: "@@undoable/undo" });
+    expect(s3).toMatchObject({ a: 0, b: 1001 });
+  });
+});
+
+describe("redo", () => {
+  const initialState = { a: 0, b: 1000 };
+  const reducer = state => {
+    return { a: state.a + 1, b: state.b + 1 };
+  };
+  let UR;
+
+  beforeEach(() => {
+    UR = undoable(reducer, ["a"]);
+  });
+
+  test("expected state modifications", () => {
+    const s1 = UR(initialState, { type: "test" });
+    expect(s1).toMatchObject({ a: 1, b: 1001 });
+
+    // verify undo->redo reverts state.
+    const s2 = UR(UR(s1, { type: "@@undoable/undo" }), {
+      type: "@@undoable/redo"
+    });
+    expect(s2).toMatchObject({ a: 1, b: 1001 });
+
+    // verify backstop when no redo future
+    const s3 = UR(s2, { type: "@@undoable/redo" });
+    expect(s3).toMatchObject({ a: 1, b: 1001 });
+  });
+
+  test("history cleared", () => {
+    // verify future cleared upon a normal state transition
+    const s1 = UR(initialState, { type: "test" });
+    expect(s1).toMatchObject({ a: 1, b: 1001 });
+    const s2 = UR(s1, { type: "@@undoable/undo" });
+    expect(s2).toMatchObject({ a: 0, b: 1001 });
+    const s3 = UR(s2, { type: "test" });
+    expect(s3).toMatchObject({ a: 1, b: 1002 });
+    const s4 = UR(s3, { type: "@@undoable/redo" });
+    expect(s4).toMatchObject({ a: 1, b: 1002 });
+  });
+});
+
+/*
+TODO:
+- historyLimit is enforced
+- action filters
+*/

client/__tests__/util/dataframe/dataframe.test.js

@@ -0,0 +1,616 @@
+import * as Dataframe from "../../../src/util/dataframe";
+
+describe("dataframe constructor", () => {
+  test("empty dataframe", () => {
+    const df = new Dataframe.Dataframe([0, 0], []);
+    expect(df).toBeDefined();
+    expect(df.dims).toEqual([0, 0]);
+    expect(df).toHaveLength(0);
+    expect(df.icol(0)).not.toBeDefined();
+  });
+
+  test("create with default indices", () => {
+    const df = new Dataframe.Dataframe(
+      [3, 2],
+      [new Int32Array(3).fill(0), new Int32Array(3).fill(1)]
+    );
+
+    expect(df).toBeDefined();
+    expect(df.dims).toEqual([3, 2]);
+    expect(df.rowIndex).toBeInstanceOf(Dataframe.IdentityInt32Index);
+    expect(df.colIndex).toBeInstanceOf(Dataframe.IdentityInt32Index);
+    expect(df.at(0, 0)).toEqual(0);
+    expect(df.at(2, 1)).toEqual(1);
+    expect(df.iat(0, 0)).toEqual(0);
+    expect(df.iat(2, 1)).toEqual(1);
+  });
+
+  test("create with labelled indices", () => {
+    const df = new Dataframe.Dataframe(
+      [3, 2],
+      [new Int32Array([0, 1, 2]), new Int32Array([3, 4, 5])],
+      new Dataframe.DenseInt32Index([2, 1, 0]),
+      new Dataframe.KeyIndex(["A", "B"])
+    );
+
+    expect(df).toBeDefined();
+    expect(df.dims).toEqual([3, 2]);
+
+    expect(df.rowIndex).toBeInstanceOf(Dataframe.DenseInt32Index);
+    expect(df.colIndex).toBeInstanceOf(Dataframe.KeyIndex);
+    expect(df.rowIndex.keys()).toEqual(new Int32Array([2, 1, 0]));
+    expect(df.colIndex.keys()).toEqual(["A", "B"]);
+
+    expect(df.at(0, "A")).toEqual(2);
+    expect(df.at(2, "B")).toEqual(3);
+    expect(df.iat(0, 0)).toEqual(0);
+    expect(df.iat(2, 1)).toEqual(5);
+  });
+});
+
+describe("simple data access", () => {
+  const df = new Dataframe.Dataframe(
+    [4, 2],
+    [
+      new Float64Array([0.0, Number.NaN, Number.POSITIVE_INFINITY, 3.14159]),
+      ["red", "blue", "green", "nan"]
+    ],
+    new Dataframe.DenseInt32Index([3, 2, 1, 0]),
+    new Dataframe.KeyIndex(["numbers", "colors"])
+  );
+
+  test("iat", () => {
+    expect(df).toBeDefined();
+
+    // present
+    expect(df.iat(0, 0)).toEqual(0.0);
+    expect(df.iat(0, 1)).toEqual("red");
+    expect(df.iat(1, 0)).toEqual(Number.NaN);
+    expect(df.iat(1, 1)).toEqual("blue");
+    expect(df.iat(2, 0)).toEqual(Number.POSITIVE_INFINITY);
+    expect(df.iat(2, 1)).toEqual("green");
+    expect(df.iat(3, 0)).toEqual(3.14159);
+    expect(df.iat(3, 1)).toEqual("nan");
+
+    // labels out of range have no defined behavior
+  });
+
+  test("at", () => {
+    expect(df).toBeDefined();
+
+    // present
+    expect(df.at(3, "numbers")).toEqual(0.0);
+    expect(df.at(3, "colors")).toEqual("red");
+    expect(df.at(2, "numbers")).toEqual(Number.NaN);
+    expect(df.at(2, "colors")).toEqual("blue");
+    expect(df.at(1, "numbers")).toEqual(Number.POSITIVE_INFINITY);
+    expect(df.at(1, "colors")).toEqual("green");
+    expect(df.at(0, "numbers")).toEqual(3.14159);
+    expect(df.at(0, "colors")).toEqual("nan");
+
+    // labels out of range have no defined behavior
+  });
+
+  test("ihas", () => {
+    expect(df).toBeDefined();
+
+    // present
+    expect(df.ihas(0, 0)).toBeTruthy();
+    expect(df.ihas(1, 1)).toBeTruthy();
+    expect(df.ihas(3, 1)).toBeTruthy();
+
+    // not present
+    expect(df.ihas(-1, -1)).toBeFalsy();
+    expect(df.ihas(0, 99)).toBeFalsy();
+    expect(df.ihas(99, 0)).toBeFalsy();
+    expect(df.ihas(99, 99)).toBeFalsy();
+    expect(df.ihas(-1, 0)).toBeFalsy();
+    expect(df.ihas(0, -1)).toBeFalsy();
+  });
+
+  test("has", () => {
+    expect(df).toBeDefined();
+
+    // present
+    expect(df.has(3, "numbers")).toBeTruthy();
+    expect(df.has(0, "numbers")).toBeTruthy();
+    expect(df.has(3, "colors")).toBeTruthy();
+    expect(df.has(0, "colors")).toBeTruthy();
+
+    // not present
+    expect(df.has(3, "foo")).toBeFalsy();
+    expect(df.has(-1, "numbers")).toBeFalsy();
+    expect(df.has(-1, -1)).toBeFalsy();
+    expect(df.has(null, null)).toBeFalsy();
+    expect(df.has(0, "foo")).toBeFalsy();
+    expect(df.has(99, "numbers")).toBeFalsy();
+    expect(df.has(99, "foo")).toBeFalsy();
+  });
+});
+
+describe("dataframe subsetting", () => {
+  describe("subset", () => {
+    const sourceDf = new Dataframe.Dataframe(
+      [3, 4],
+      [
+        new Int32Array([0, 1, 2]),
+        ["A", "B", "C"],
+        new Float32Array([4.4, 5.5, 6.6]),
+        ["red", "green", "blue"]
+      ],
+      null,
+      new Dataframe.KeyIndex(["int32", "string", "float32", "colors"])
+    );
+
+    test("all rows, one column", () => {
+      const dfA = sourceDf.subset(null, ["colors"]);
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([3, 1]);
+      expect(dfA.iat(0, 0)).toEqual("red");
+      expect(dfA.at(2, "colors")).toEqual("blue");
+      expect(dfA.col("colors").asArray()).toEqual(["red", "green", "blue"]);
+      expect(dfA.icol(0).asArray()).toEqual(["red", "green", "blue"]);
+      expect(dfA.col("colors").asArray()).toEqual(
+        sourceDf.col("colors").asArray()
+      );
+      expect(dfA.rowIndex.keys()).toEqual(sourceDf.rowIndex.keys());
+      expect(dfA.colIndex.keys()).toEqual(["colors"]);
+    });
+
+    test("all rows, two columns", () => {
+      const dfB = sourceDf.subset(null, ["colors", "float32"]);
+      expect(dfB).toBeDefined();
+      expect(dfB.dims).toEqual([3, 2]);
+      expect(dfB.iat(0, 0)).toBeCloseTo(4.4);
+      expect(dfB.iat(0, 1)).toEqual("red");
+      expect(dfB.at(2, "colors")).toEqual("blue");
+      expect(dfB.at(2, "float32")).toBeCloseTo(6.6);
+      expect(dfB.col("colors").asArray()).toEqual(["red", "green", "blue"]);
+      expect(dfB.col("float32").asArray()).toEqual(
+        new Float32Array([4.4, 5.5, 6.6])
+      );
+      expect(dfB.icol(0).asArray()).toEqual(dfB.col("float32").asArray());
+      expect(dfB.icol(1).asArray()).toEqual(dfB.col("colors").asArray());
+      expect(dfB.col("colors").asArray()).toEqual(
+        sourceDf.col("colors").asArray()
+      );
+      expect(dfB.col("float32").asArray()).toEqual(
+        sourceDf.col("float32").asArray()
+      );
+      expect(dfB.rowIndex.keys()).toEqual(sourceDf.rowIndex.keys());
+      expect(dfB.colIndex.keys()).toEqual(["float32", "colors"]);
+    });
+
+    test("one row, all columns", () => {
+      const dfC = sourceDf.subset([1], null);
+      expect(dfC).toBeDefined();
+      expect(dfC.dims).toEqual([1, 4]);
+      expect(dfC.iat(0, 0)).toEqual(1);
+      expect(dfC.iat(0, 1)).toEqual("B");
+      expect(dfC.iat(0, 2)).toBeCloseTo(5.5);
+      expect(dfC.iat(0, 3)).toEqual("green");
+      expect(dfC.rowIndex.keys()).toEqual(new Int32Array([1]));
+      expect(dfC.colIndex.keys()).toEqual(sourceDf.colIndex.keys());
+    });
+
+    test("two rows, all columns", () => {
+      const dfD = sourceDf.subset([0, 2], null);
+      expect(dfD).toBeDefined();
+      expect(dfD.dims).toEqual([2, 4]);
+      expect(dfD.icol(0).asArray()).toEqual(new Int32Array([0, 2]));
+      expect(dfD.icol(1).asArray()).toEqual(["A", "C"]);
+      expect(dfD.icol(2).asArray()).toEqual(new Float32Array([4.4, 6.6]));
+      expect(dfD.icol(3).asArray()).toEqual(["red", "blue"]);
+      expect(dfD.rowIndex.keys()).toEqual(new Int32Array([0, 2]));
+      expect(dfD.colIndex.keys()).toEqual(sourceDf.colIndex.keys());
+    });
+
+    test("all rows, all columns", () => {
+      const dfE = sourceDf.subset(null, null);
+      expect(dfE).toBeDefined();
+      expect(dfE.dims).toEqual([3, 4]);
+      expect(dfE.icol(0).asArray()).toEqual(sourceDf.icol(0).asArray());
+      expect(dfE.icol(1).asArray()).toEqual(sourceDf.icol(1).asArray());
+      expect(dfE.icol(2).asArray()).toEqual(sourceDf.icol(2).asArray());
+      expect(dfE.icol(3).asArray()).toEqual(sourceDf.icol(3).asArray());
+      expect(dfE.rowIndex.keys()).toEqual(sourceDf.rowIndex.keys());
+      expect(dfE.colIndex.keys()).toEqual(sourceDf.colIndex.keys());
+    });
+
+    test("two rows, two colums", () => {
+      const dfF = sourceDf.subset([0, 2], ["int32", "float32"]);
+      expect(dfF).toBeDefined();
+      expect(dfF.dims).toEqual([2, 2]);
+      expect(dfF.icol(0).asArray()).toEqual(new Int32Array([0, 2]));
+      expect(dfF.icol(1).asArray()).toEqual(new Float32Array([4.4, 6.6]));
+      expect(dfF.rowIndex.keys()).toEqual(new Int32Array([0, 2]));
+      expect(dfF.colIndex.keys()).toEqual(["int32", "float32"]);
+    });
+
+    test("withRowIndex", () => {
+      const df = sourceDf.subset(
+        null,
+        ["int32", "float32"],
+        new Dataframe.DenseInt32Index([3, 2, 1])
+      );
+      expect(df.colIndex).toBeInstanceOf(Dataframe.KeyIndex);
+      expect(df.rowIndex).toBeInstanceOf(Dataframe.DenseInt32Index);
+      expect(df.at(3, "int32")).toEqual(df.iat(0, 0));
+    });
+
+    test("withRowIndex error checks", () => {
+      expect(() =>
+        sourceDf.subset(null, ["red"], new Dataframe.IdentityInt32Index(1))
+      ).toThrow(RangeError);
+      expect(() =>
+        sourceDf.subset(null, ["red"], new Dataframe.DenseInt32Index([0, 1]))
+      ).toThrow(RangeError);
+      expect(() =>
+        sourceDf.subset(null, ["red"], new Dataframe.KeyIndex([0, 1, 2, 3]))
+      ).toThrow(RangeError);
+    });
+  });
+
+  test("isubsetMask", () => {
+    const sourceDf = new Dataframe.Dataframe(
+      [3, 4],
+      [
+        new Int32Array([0, 1, 2]),
+        ["A", "B", "C"],
+        new Float32Array([4.4, 5.5, 6.6]),
+        ["red", "green", "blue"]
+      ],
+      new Dataframe.DenseInt32Index([2, 4, 6]),
+      new Dataframe.KeyIndex(["int32", "string", "float32", "colors"])
+    );
+
+    const dfA = sourceDf.isubsetMask(
+      new Uint8Array([0, 1, 1]),
+      new Uint8Array([1, 0, 0, 1])
+    );
+    expect(dfA.dims).toEqual([2, 2]);
+    expect(dfA.icol(0).asArray()).toEqual(new Int32Array([1, 2]));
+    expect(dfA.icol(1).asArray()).toEqual(["green", "blue"]);
+    expect(dfA.rowIndex.keys()).toEqual(new Int32Array([4, 6]));
+    expect(dfA.colIndex.keys()).toEqual(["int32", "colors"]);
+  });
+});
+
+describe("dataframe factories", () => {
+  test("create", () => {
+    const df = Dataframe.Dataframe.create(
+      [3, 3],
+      [
+        new Array(3).fill(0),
+        new Int16Array(3).fill(99),
+        new Float64Array(3).fill(1.1)
+      ]
+    );
+
+    expect(df).toBeDefined();
+    expect(df.dims).toEqual([3, 3]);
+    expect(df).toHaveLength(3);
+    expect(df.iat(0, 0)).toEqual(0);
+    expect(df.iat(1, 1)).toEqual(99);
+    expect(df.iat(2, 2)).toBeCloseTo(1.1);
+    expect(df.iat(0, 0)).toEqual(df.at(0, 0));
+    expect(df.iat(1, 1)).toEqual(df.at(1, 1));
+    expect(df.iat(2, 2)).toEqual(df.at(2, 2));
+  });
+
+  test("clone", () => {
+    const dfA = new Dataframe.Dataframe(
+      [3, 2],
+      [new Int32Array([0, 1, 2]), new Int32Array([3, 4, 5])],
+      new Dataframe.DenseInt32Index([2, 1, 0]),
+      new Dataframe.KeyIndex(["A", "B"])
+    );
+
+    const dfB = dfA.clone();
+    expect(dfB).not.toBe(dfA);
+    expect(dfB.dims).toEqual(dfA.dims);
+    expect(dfB).toHaveLength(dfA.length);
+    expect(dfB.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    expect(dfB.colIndex.keys()).toEqual(dfA.colIndex.keys());
+    for (let i = 0, l = dfB.dims[1]; i < l; i += 1) {
+      expect(dfB.icol(i).asArray()).toEqual(dfA.icol(i).asArray());
+    }
+  });
+
+  describe("withCol", () => {
+    test("KeyIndex", () => {
+      const df = new Dataframe.Dataframe(
+        [2, 2],
+        [["red", "blue"], [true, false]],
+        null,
+        new Dataframe.KeyIndex(["colors", "bools"])
+      );
+      const dfA = df.withCol("numbers", [1, 0]);
+
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([2, 3]);
+      expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.icol(1).asArray()).toEqual([true, false]);
+      expect(dfA.icol(2).asArray()).toEqual([1, 0]);
+      expect(dfA.col("numbers").asArray()).toEqual([1, 0]);
+      expect(dfA.colIndex.keys()).toEqual(["colors", "bools", "numbers"]);
+      expect(df.colIndex.keys()).toEqual(["colors", "bools"]);
+      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    });
+
+    test("DenseInt32Index", () => {
+      const df = new Dataframe.Dataframe(
+        [2, 2],
+        [["red", "blue"], [true, false]],
+        null,
+        new Dataframe.DenseInt32Index([74, 75])
+      );
+      const dfA = df.withCol(72, [1, 0]);
+
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([2, 3]);
+      expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.icol(1).asArray()).toEqual([true, false]);
+      expect(dfA.icol(2).asArray()).toEqual([1, 0]);
+      expect(dfA.col(74).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.col(75).asArray()).toEqual([true, false]);
+      expect(dfA.col(72).asArray()).toEqual([1, 0]);
+      expect(dfA.colIndex.keys()).toEqual(new Int32Array([74, 75, 72]));
+      expect(df.colIndex.keys()).toEqual(new Int32Array([74, 75]));
+      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    });
+
+    test("DenseInt32Index promote", () => {
+      const df = new Dataframe.Dataframe(
+        [2, 2],
+        [["red", "blue"], [true, false]],
+        null,
+        new Dataframe.DenseInt32Index([74, 75])
+      );
+      const dfA = df.withCol(999, [1, 0]);
+
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([2, 3]);
+      expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.icol(1).asArray()).toEqual([true, false]);
+      expect(dfA.icol(2).asArray()).toEqual([1, 0]);
+      expect(dfA.col(74).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.col(75).asArray()).toEqual([true, false]);
+      expect(dfA.col(999).asArray()).toEqual([1, 0]);
+      expect(dfA.colIndex.keys()).toEqual(new Int32Array([74, 75, 999]));
+      expect(df.colIndex.keys()).toEqual(new Int32Array([74, 75]));
+      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    });
+
+    test("IdentityInt32Index with last", () => {
+      const df = new Dataframe.Dataframe(
+        [2, 2],
+        [["red", "blue"], [true, false]],
+        null,
+        null
+      );
+      const dfA = df.withCol(2, [1, 0]);
+
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([2, 3]);
+      expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.icol(1).asArray()).toEqual([true, false]);
+      expect(dfA.icol(2).asArray()).toEqual([1, 0]);
+      expect(dfA.col(0).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.col(1).asArray()).toEqual([true, false]);
+      expect(dfA.col(2).asArray()).toEqual([1, 0]);
+      expect(dfA.colIndex.keys()).toEqual(new Int32Array([0, 1, 2]));
+      expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1]));
+      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    });
+
+    test("IdentityInt32Index promote", () => {
+      const df = new Dataframe.Dataframe(
+        [2, 2],
+        [["red", "blue"], [true, false]],
+        null,
+        null
+      );
+      const dfA = df.withCol(99, [1, 0]);
+
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([2, 3]);
+      expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.icol(1).asArray()).toEqual([true, false]);
+      expect(dfA.icol(2).asArray()).toEqual([1, 0]);
+      expect(dfA.col(0).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.col(1).asArray()).toEqual([true, false]);
+      expect(dfA.col(99).asArray()).toEqual([1, 0]);
+      expect(dfA.colIndex.keys()).toEqual(new Int32Array([0, 1, 99]));
+      expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1]));
+      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    });
+
+    describe("handle column dimensions correctly", () => {
+      /*
+      there are two conditions:
+      - empty dataframe - will accept an add of any dimensionality
+      - non-empty dataframe - added column must match row-count dimension
+      */
+      test("empty.withCol", () => {
+        const edf = Dataframe.Dataframe.empty();
+        const df = edf.withCol("foo", [1, 2, 3]);
+
+        expect(edf).toBeDefined();
+        expect(df).toBeDefined();
+        expect(edf).not.toEqual(df);
+        expect(df.dims).toEqual([3, 1]);
+        expect(df.icol(0).asArray()).toEqual([1, 2, 3]);
+      });
+
+      test("withCol dimension check", () => {
+        const dfA = new Dataframe.Dataframe([1, 1], [["a"]]);
+        expect(() => {
+          dfA.withCol(1, []);
+        }).toThrow(RangeError);
+      });
+    });
+  });
+
+  describe("dropCol", () => {
+    test("KeyIndex", () => {
+      const df = new Dataframe.Dataframe(
+        [2, 3],
+        [["red", "blue"], [true, false], [1, 0]],
+        null,
+        new Dataframe.KeyIndex(["colors", "bools", "numbers"])
+      );
+      const dfA = df.dropCol("colors");
+
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([2, 2]);
+      expect(dfA.icol(0).asArray()).toEqual([true, false]);
+      expect(dfA.icol(1).asArray()).toEqual([1, 0]);
+      expect(dfA.col("numbers").asArray()).toEqual([1, 0]);
+      expect(dfA.colIndex.keys()).toEqual(["bools", "numbers"]);
+      expect(df.colIndex.keys()).toEqual(["colors", "bools", "numbers"]);
+      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    });
+
+    test("IdentityInt32Index drop first", () => {
+      const df = new Dataframe.Dataframe(
+        [2, 3],
+        [["red", "blue"], [true, false], [1, 0]],
+        null,
+        null
+      );
+      const dfA = df.dropCol(0);
+
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([2, 2]);
+      expect(dfA.icol(0).asArray()).toEqual([true, false]);
+      expect(dfA.icol(1).asArray()).toEqual([1, 0]);
+      expect(df.col(1).asArray()).toEqual(dfA.col(1).asArray());
+      expect(df.col(2).asArray()).toEqual(dfA.col(2).asArray());
+      expect(dfA.colIndex.keys()).toEqual(new Int32Array([1, 2]));
+      expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1, 2]));
+      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    });
+
+    test("IdentityInt32Index drop last", () => {
+      const df = new Dataframe.Dataframe(
+        [2, 3],
+        [["red", "blue"], [true, false], [1, 0]],
+        null,
+        null
+      );
+      const dfA = df.dropCol(2);
+
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([2, 2]);
+      expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.icol(1).asArray()).toEqual([true, false]);
+      expect(df.col(0).asArray()).toEqual(dfA.col(0).asArray());
+      expect(df.col(1).asArray()).toEqual(dfA.col(1).asArray());
+      expect(dfA.colIndex.keys()).toEqual(new Int32Array([0, 1]));
+      expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1, 2]));
+      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    });
+
+    test("DenseInt32Index", () => {
+      const df = new Dataframe.Dataframe(
+        [2, 3],
+        [["red", "blue"], [true, false], [1, 0]],
+        null,
+        new Dataframe.DenseInt32Index([102, 101, 100])
+      );
+      const dfA = df.dropCol(101);
+
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([2, 2]);
+      expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.icol(1).asArray()).toEqual([1, 0]);
+      expect(dfA.col(100).asArray()).toEqual([1, 0]);
+      expect(dfA.col(102).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.colIndex.keys()).toEqual(new Int32Array([102, 100]));
+      expect(df.colIndex.keys()).toEqual(new Int32Array([102, 101, 100]));
+      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    });
+  });
+});
+
+describe("dataframe col", () => {
+  let df = null;
+  beforeEach(() => {
+    df = new Dataframe.Dataframe(
+      [2, 2],
+      [[true, false], [1, 0]],
+      null,
+      new Dataframe.KeyIndex(["A", "B"])
+    );
+  });
+
+  test("col", () => {
+    expect(df).toBeDefined();
+    expect(df.col("A")).toBe(df.icol(0));
+    expect(df.col("B")).toBe(df.icol(1));
+    expect(df.col("undefined")).toBeUndefined();
+    expect(df.icol("undefined")).toBeUndefined();
+
+    const colA = df.col("A");
+    expect(colA).toBeInstanceOf(Function);
+    expect(colA.asArray).toBeInstanceOf(Function);
+    expect(colA.has).toBeInstanceOf(Function);
+    expect(colA.ihas).toBeInstanceOf(Function);
+    expect(colA.indexOf).toBeInstanceOf(Function);
+    expect(colA.iget).toBeInstanceOf(Function);
+  });
+
+  test("col.asArray", () => {
+    expect(df).toBeDefined();
+    expect(df.col("A").asArray()).toEqual([true, false]);
+    expect(df.icol(0).asArray()).toEqual([true, false]);
+    expect(df.col("B").asArray()).toEqual([1, 0]);
+    expect(df.icol(1).asArray()).toEqual([1, 0]);
+  });
+
+  test("col.has", () => {
+    expect(df).toBeDefined();
+    expect(df.col("A").has(-1)).toBe(false);
+    expect(df.col("A").has(0)).toBe(true);
+    expect(df.col("A").has(1)).toBe(true);
+    expect(df.col("A").has(2)).toBe(false);
+    expect(df.col("B").has(-1)).toBe(false);
+    expect(df.col("B").has(0)).toBe(true);
+    expect(df.col("B").has(1)).toBe(true);
+    expect(df.col("B").has(2)).toBe(false);
+  });
+
+  test("col.ihas", () => {
+    expect(df).toBeDefined();
+    expect(df.col("A").ihas(-1)).toBe(false);
+    expect(df.col("A").ihas(0)).toBe(true);
+    expect(df.col("A").ihas(1)).toBe(true);
+    expect(df.col("A").ihas(2)).toBe(false);
+    expect(df.col("B").ihas(-1)).toBe(false);
+    expect(df.col("B").ihas(0)).toBe(true);
+    expect(df.col("B").ihas(1)).toBe(true);
+    expect(df.col("B").ihas(2)).toBe(false);
+  });
+
+  test("col.iget", () => {
+    expect(df).toBeDefined();
+    expect(df.col("A").iget(0)).toEqual(df.iat(0, 0));
+    expect(df.col("B").iget(1)).toEqual(df.iat(1, 1));
+  });
+
+  test("col.indexOf", () => {
+    expect(df).toBeDefined();
+    expect(df.col("A").indexOf(true)).toEqual(0);
+    expect(df.col("A").indexOf(false)).toEqual(1);
+    expect(df.col("A").indexOf(99)).toBeUndefined();
+    expect(df.col("A").indexOf(undefined)).toBeUndefined();
+    expect(df.col("A").indexOf(1)).toBeUndefined();
+
+    expect(df.col("B").indexOf(1)).toEqual(0);
+    expect(df.col("B").indexOf(0)).toEqual(1);
+    expect(df.col("B").indexOf(99)).toBeUndefined();
+    expect(df.col("B").indexOf(undefined)).toBeUndefined();
+    expect(df.col("B").indexOf(true)).toBeUndefined();
+  });
+});

client/__tests__/util/dataframe/summarize.test.js

@@ -0,0 +1,253 @@
+import * as Dataframe from "../../../src/util/dataframe";
+
+function float32Conversion(f) {
+  return new Float32Array([f])[0];
+}
+
+describe("Dataframe column summary", () => {
+  test("empty column test", () => {
+    const df = Dataframe.Dataframe.create([0, 1], [[]]);
+    const summary = df.icol(0).summarize();
+    expect(summary).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: [],
+        categoryCounts: new Map(),
+        numCategories: 0
+      })
+    );
+  });
+
+  test("simple test", () => {
+    const df = new Dataframe.Dataframe(
+      [1, 6],
+      [
+        ["n1"],
+        ["hi"],
+        [true],
+        new Float32Array([39.3]),
+        new Int32Array([99]),
+        [1]
+      ],
+      null,
+      new Dataframe.KeyIndex([
+        "name",
+        "nameString",
+        "nameBoolean",
+        "nameFloat32",
+        "nameInt32",
+        "nameCategorical"
+      ])
+    );
+
+    expect(df.icol(0).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: ["n1"],
+        categoryCounts: new Map([["n1", 1]]),
+        numCategories: 1
+      })
+    );
+    expect(df.icol(1).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: ["hi"],
+        categoryCounts: new Map([["hi", 1]]),
+        numCategories: 1
+      })
+    );
+    expect(df.icol(2).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: [true],
+        categoryCounts: new Map([[true, 1]]),
+        numCategories: 1
+      })
+    );
+    expect(df.icol(3).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: false,
+        min: float32Conversion(39.3),
+        max: float32Conversion(39.3),
+        nan: 0,
+        ninf: 0,
+        pinf: 0
+      })
+    );
+    expect(df.icol(4).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: false,
+        min: 99,
+        max: 99,
+        nan: 0,
+        ninf: 0,
+        pinf: 0
+      })
+    );
+    expect(df.icol(5).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: [1],
+        categoryCounts: new Map([[1, 1]]),
+        numCategories: 1
+      })
+    );
+  });
+
+  test("multi test", () => {
+    const df = new Dataframe.Dataframe(
+      [3, 6],
+      [
+        ["n0", "n1", "n2"],
+        ["hi", "hi", "bye"],
+        [false, true, true],
+        new Float32Array([39.3, 39.3, 0]),
+        new Int32Array([99, 99, 99]),
+        [1, false, "0"]
+      ],
+      null,
+      new Dataframe.KeyIndex([
+        "name",
+        "nameString",
+        "nameBoolean",
+        "nameFloat32",
+        "nameInt32",
+        "nameCategorical"
+      ])
+    );
+
+    expect(df.icol(0).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining(["n0", "n1", "n2"]),
+        categoryCounts: new Map([["n0", 1], ["n1", 1], ["n2", 1]]),
+        numCategories: 3
+      })
+    );
+    expect(df.icol(1).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining(["hi", "bye"]),
+        categoryCounts: new Map([["hi", 2], ["bye", 1]]),
+        numCategories: 2
+      })
+    );
+    expect(df.icol(2).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining([true, false]),
+        categoryCounts: new Map([[true, 2], [false, 1]]),
+        numCategories: 2
+      })
+    );
+    expect(df.icol(3).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: false,
+        min: 0,
+        max: float32Conversion(39.3),
+        nan: 0,
+        ninf: 0,
+        pinf: 0
+      })
+    );
+    expect(df.icol(4).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: false,
+        min: 99,
+        max: 99,
+        nan: 0,
+        ninf: 0,
+        pinf: 0
+      })
+    );
+    expect(df.icol(5).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining([1, false, "0"]),
+        categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]),
+        numCategories: 3
+      })
+    );
+  });
+
+  test("non-finite numbers", () => {
+    const df = new Dataframe.Dataframe(
+      [4, 6],
+      [
+        ["n0", "n1", "n2", "n2"],
+        ["hi", "hi", "bye", "bye"],
+        [false, true, true, true],
+        new Float32Array([
+          39.3,
+          Number.NEGATIVE_INFINITY,
+          Number.NaN,
+          Number.POSITIVE_INFINITY
+        ]),
+        new Int32Array([99, 99, 99, 99]),
+        [1, false, "0", "0"]
+      ],
+      null,
+      new Dataframe.KeyIndex([
+        "name",
+        "nameString",
+        "nameBoolean",
+        "nameFloat32",
+        "nameInt32",
+        "nameCategorical"
+      ])
+    );
+
+    expect(df.icol(0).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining(["n0", "n1", "n2"]),
+        categoryCounts: new Map([["n0", 1], ["n1", 1], ["n2", 2]]),
+        numCategories: 3
+      })
+    );
+    expect(df.icol(1).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining(["hi", "bye"]),
+        categoryCounts: new Map([["hi", 2], ["bye", 1]]),
+        numCategories: 2
+      })
+    );
+    expect(df.icol(2).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining([true, false]),
+        categoryCounts: new Map([[true, 2], [false, 1]]),
+        numCategories: 2
+      })
+    );
+    expect(df.icol(3).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: false,
+        min: float32Conversion(39.3),
+        max: float32Conversion(39.3),
+        nan: 1,
+        ninf: 1,
+        pinf: 1
+      })
+    );
+    expect(df.icol(4).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: false,
+        min: 99,
+        max: 99,
+        nan: 0,
+        ninf: 0,
+        pinf: 0
+      })
+    );
+    expect(df.icol(5).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining([1, false, "0"]),
+        categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]),
+        numCategories: 3
+      })
+    );
+  });
+});

client/__tests__/util/stateManager/keyvalcache.test.js

@@ -1,249 +0,0 @@
-import _ from "lodash";
-import * as kvCache from "../../../src/util/stateManager/keyvalcache";
-
-/*
-This is PRIVATE to keyvalcache and must be kept in sync with
-any changs ot that module.  Need to Know - to enable error handling test
-*/
-const cachePrivateKey = "__kvcachekey__";
-
-/*
-helper function - promisify setTimeout()
-*/
-function timeout(ms) {
-  return new Promise(resolve => setTimeout(resolve, ms));
-}
-
-describe("kvcache API", () => {
-  /*
-  test the happy path create/set/get API
-  */
-
-  test("simple create", () => {
-    /* with defaults */
-    const kvc = kvCache.create();
-    expect(kvc).toBeDefined();
-    expect(kvc).toEqual(expect.objectContaining({}));
-    expect(kvCache.get(kvc, "test")).toBeUndefined();
-
-    /* with params */
-    const kvc1 = kvCache.create(/* lowWatermark */ 99, /* minTTL */ 0);
-    expect(kvc1).toBeDefined();
-    expect(kvc1).toEqual(expect.objectContaining({}));
-  });
-
-  test("set/get", () => {
-    /*
-    - check basic get/set functionality
-    - check set does not mutate source cache
-    */
-    const keyName = "foo";
-    const kvc1 = kvCache.create();
-    expect(kvc1).toBeDefined();
-    expect(kvCache.get(kvc1, keyName)).toBeUndefined();
-
-    const val2 = [2];
-    const kvc2 = kvCache.set(kvc1, keyName, val2);
-    expect(kvc2).toBeDefined();
-    expect(kvc2).not.toBe(kvc1);
-    expect(kvCache.get(kvc1, keyName)).toBeUndefined();
-    expect(kvCache.get(kvc2, keyName)).toBe(val2);
-
-    const val3 = [3];
-    const kvc3 = kvCache.set(kvc2, keyName, val3);
-    expect(kvc3).toBeDefined();
-    expect(kvc3).not.toBe(kvc1);
-    expect(kvc3).not.toBe(kvc2);
-    expect(kvCache.get(kvc1, keyName)).toBeUndefined();
-    expect(kvCache.get(kvc2, keyName)).toBe(val2);
-    expect(kvCache.get(kvc3, keyName)).toBe(val3);
-  });
-});
-
-describe("common error handling", () => {
-  /*
-  Test common error handlers
-  */
-
-  test("set() protection from namespace pollution", () => {
-    /*
-    Test that set() will not allow use of the private cache key
-    */
-    const kvc = kvCache.create();
-    expect(() => {
-      kvCache.set(kvc, cachePrivateKey, {});
-    }).toThrow();
-  });
-
-  test("create() does not accept bogus config", () => {
-    expect(() => {
-      kvCache.create([], {});
-    }).toThrow();
-    expect(() => {
-      kvCache.create(-99, 0);
-    }).toThrow();
-    expect(() => {
-      kvCache.create(100, -1);
-    }).toThrow();
-    expect(() => {
-      kvCache.create(1000, "foobar");
-    }).toThrow();
-    expect(() => {
-      kvCache.create(null, 8);
-    }).toThrow();
-  });
-});
-
-describe("map", () => {
-  /*
-  Test kvCache.map() - create new cache that is a transformation of an
-  existing cache
-  */
-  test("map of empty cache", () => {
-    const kvc = kvCache.create();
-    const callback = jest.fn();
-    const kvcMapped = kvCache.map(kvc, callback);
-    expect(callback).not.toHaveBeenCalled();
-    expect(kvcMapped).toBeDefined();
-    expect(kvcMapped).not.toBe(kvc); // immutable operation
-    expect(kvcMapped).toEqual(kvc);
-  });
-
-  test("map of non-empty cache", () => {
-    const key = "aKey";
-    const val = [0, 1, 2];
-    let kvc = kvCache.create();
-    kvc = kvCache.set(kvc, key, val);
-    const mockCB = jest.fn().mockImplementation(v => [...v]);
-    const kvcMapped = kvCache.map(kvc, mockCB);
-
-    expect(kvcMapped).toBeDefined();
-    expect(kvcMapped).not.toBe(kvc); // immutable operation
-    expect(_.isEqual(kvc, kvcMapped)).toBe(true);
-
-    expect(mockCB).toHaveBeenCalledTimes(1);
-    expect(mockCB).toHaveBeenLastCalledWith(val, key);
-  });
-});
-
-describe("flush", () => {
-  /*
-  test various cache flush behavior
-  */
-  test("flush - lowWatermark, disable minTTL", () => {
-    /*
-    verify lowWatermark functions correctly
-    */
-
-    // set lowWatermark to 2, set three times - only the final two
-    // should remain.
-    let kvc = kvCache.create(2, 0);
-    ["a", "b", "c"].forEach(k => {
-      kvc = kvCache.set(kvc, k, []);
-    });
-
-    expect(kvc).toEqual(
-      expect.objectContaining({
-        b: expect.arrayContaining([]),
-        c: expect.arrayContaining([])
-      })
-    );
-    expect(kvc).toEqual(
-      expect.not.objectContaining({
-        a: expect.arrayContaining([])
-      })
-    );
-  });
-
-  test("flush - minTTL, disable lowWatermark", async () => {
-    /*
-    verify minTTL functions correctly
-    */
-
-    // set minTTL to 1 ms
-    let kvc = kvCache.create(0, 10);
-    kvc = kvCache.set(kvc, "a", []);
-    await timeout(20);
-    ["b", "c"].forEach(k => {
-      kvc = kvCache.set(kvc, k, []);
-    });
-
-    expect(kvc).toEqual(
-      expect.objectContaining({
-        b: expect.arrayContaining([]),
-        c: expect.arrayContaining([])
-      })
-    );
-    expect(kvc).toEqual(
-      expect.not.objectContaining({
-        a: expect.arrayContaining([])
-      })
-    );
-  });
-
-  test("flush - minTTL and lowWatermark", async () => {
-    /*
-    verify minTTL functions correctly
-    */
-
-    // set lowwatermark to 3, minTTL to 1 ms
-    let kvc = kvCache.create(3, 10);
-    kvc = kvCache.set(kvc, "a", []);
-    // delay
-    await timeout(20);
-    ["b", "c"].forEach(k => {
-      kvc = kvCache.set(kvc, k, []);
-    });
-
-    expect(kvc).toEqual(
-      expect.objectContaining({
-        a: expect.arrayContaining([]),
-        b: expect.arrayContaining([]),
-        c: expect.arrayContaining([])
-      })
-    );
-
-    kvc = kvCache.set(kvc, "d", []);
-    expect(kvc).toEqual(
-      expect.objectContaining({
-        b: expect.arrayContaining([]),
-        c: expect.arrayContaining([]),
-        d: expect.arrayContaining([])
-      })
-    );
-    expect(kvc).toEqual(
-      expect.not.objectContaining({
-        a: expect.arrayContaining([])
-      })
-    );
-  });
-
-  test("manual flush", async () => {
-    let kvc = kvCache.create(1, 10);
-    ["a", "b", "c", "d"].forEach(k => {
-      kvc = kvCache.set(kvc, k, []);
-    });
-
-    // Before TTL has expired, should have all values in cache.
-    expect(kvc).toEqual(
-      expect.objectContaining({
-        a: expect.arrayContaining([]),
-        b: expect.arrayContaining([]),
-        c: expect.arrayContaining([])
-      })
-    );
-
-    // let TTL expire
-    await timeout(10);
-
-    // manually flush
-    const postFlushKvc = kvCache.flush(kvc);
-    expect(postFlushKvc).toBeDefined();
-    expect(postFlushKvc).not.toBe(kvc);
-    expect(postFlushKvc).toEqual(
-      expect.objectContaining({
-        d: expect.arrayContaining([])
-      })
-    );
-  });
-});

client/__tests__/util/stateManager/sampleResponses.js

@@ -157,16 +157,6 @@ const anAnnotationsVarFBSResponse = (() => {
   return encodeMatrix(columns, anAnnotationsVarJSONResponse.names);
 })();
 
-const aLayoutJSONResponse = {
-  layout: {
-    ndims: 2,
-    coordinates: _()
-      .range(nObs)
-      .map(idx => [idx, Math.random(), Math.random()])
-      .value()
-  }
-};
-
 const aLayoutFBSResponse = (() => {
   const coords = [
     new Float32Array(nObs).fill(Math.random()),
@@ -190,7 +180,7 @@ const aLayoutFBSResponse = (() => {
 
   NetEncoding.Matrix.startMatrix(builder);
   NetEncoding.Matrix.addNRows(builder, nObs);
-  NetEncoding.Matrix.addNCols(builder, nVar);
+  NetEncoding.Matrix.addNCols(builder, coords.length);
   NetEncoding.Matrix.addColumns(builder, columns);
   const matrix = NetEncoding.Matrix.endMatrix(builder);
   builder.finish(matrix);

client/__tests__/util/stateManager/summarizeAnnotations.test.js

@@ -1,280 +0,0 @@
-import summarizeAnnotations from "../../../src/util/stateManager/summarizeAnnotations";
-
-describe("summarizeAnnotations", () => {
-  const schema = {
-    annotations: {
-      obs: [
-        { name: "name", type: "string" },
-        { name: "nameString", type: "string" },
-        { name: "nameBoolean", type: "boolean" },
-        { name: "nameFloat32", type: "float32" },
-        { name: "nameInt32", type: "int32" },
-        {
-          name: "nameCategorical",
-          type: "categorical",
-          categories: [true, false, 1, 0, 0.00001, 4383.4833, "test", "", "0"]
-        }
-      ],
-      var: [{ name: "name", type: "string" }]
-    }
-  };
-
-  test("empty test", () => {
-    const summary = summarizeAnnotations(schema, [], []);
-    expect(summary).toEqual(
-      expect.objectContaining({
-        obs: {
-          nameString: {
-            categorical: true,
-            categories: [],
-            categoryCounts: new Map(),
-            numCategories: 0
-          },
-          nameBoolean: {
-            categorical: true,
-            categories: [],
-            categoryCounts: new Map(),
-            numCategories: 0
-          },
-          nameFloat32: {
-            categorical: false,
-            range: {
-              max: undefined,
-              min: undefined,
-              nan: 0,
-              ninf: 0,
-              pinf: 0
-            }
-          },
-          nameInt32: {
-            categorical: false,
-            range: {
-              max: undefined,
-              min: undefined,
-              nan: 0,
-              ninf: 0,
-              pinf: 0
-            }
-          },
-          nameCategorical: {
-            categorical: true,
-            categories: [],
-            categoryCounts: new Map(),
-            numCategories: 0
-          }
-        },
-        var: {}
-      })
-    );
-  });
-
-  test("simple test", () => {
-    const obsAnnotations = [
-      {
-        __index__: 0,
-        name: "n1",
-        nameString: "hi",
-        nameBoolean: true,
-        nameFloat32: 39.3,
-        nameInt32: 99,
-        nameCategorical: 1
-      }
-    ];
-    const varAnnotations = [];
-
-    const summary = summarizeAnnotations(
-      schema,
-      obsAnnotations,
-      varAnnotations
-    );
-
-    expect(summary).toEqual(
-      expect.objectContaining({
-        obs: {
-          nameString: {
-            categorical: true,
-            categories: ["hi"],
-            categoryCounts: new Map([["hi", 1]]),
-            numCategories: 1
-          },
-          nameBoolean: {
-            categorical: true,
-            categories: [true],
-            categoryCounts: new Map([[true, 1]]),
-            numCategories: 1
-          },
-          nameFloat32: {
-            categorical: false,
-            range: { min: 39.3, max: 39.3, nan: 0, ninf: 0, pinf: 0 }
-          },
-          nameInt32: {
-            categorical: false,
-            range: { min: 99, max: 99, nan: 0, ninf: 0, pinf: 0 }
-          },
-          nameCategorical: {
-            categorical: true,
-            categories: [1],
-            categoryCounts: new Map([[1, 1]]),
-            numCategories: 1
-          }
-        },
-        var: {}
-      })
-    );
-  });
-
-  test("multi test", () => {
-    const obsAnnotations = [
-      {
-        __index__: 0,
-        name: "n0",
-        nameString: "hi",
-        nameBoolean: false,
-        nameFloat32: 39.3,
-        nameInt32: 99,
-        nameCategorical: 1
-      },
-      {
-        __index__: 1,
-        name: "n1",
-        nameString: "hi",
-        nameBoolean: true,
-        nameFloat32: 39.3,
-        nameInt32: 99,
-        nameCategorical: false
-      },
-      {
-        __index__: 2,
-        name: "n2",
-        nameString: "bye",
-        nameBoolean: true,
-        nameFloat32: 0,
-        nameInt32: 99,
-        nameCategorical: "0"
-      }
-    ];
-    const varAnnotations = [];
-
-    const summary = summarizeAnnotations(
-      schema,
-      obsAnnotations,
-      varAnnotations
-    );
-
-    expect(summary).toMatchObject(
-      expect.objectContaining({
-        obs: {
-          nameString: {
-            categorical: true,
-            categories: expect.arrayContaining(["hi", "bye"]),
-            categoryCounts: new Map([["hi", 2], ["bye", 1]]),
-            numCategories: 2
-          },
-          nameBoolean: {
-            categorical: true,
-            categories: expect.arrayContaining([true, false]),
-            categoryCounts: new Map([[true, 2], [false, 1]]),
-            numCategories: 2
-          },
-          nameFloat32: {
-            categorical: false,
-            range: { min: 0, max: 39.3, nan: 0, ninf: 0, pinf: 0 }
-          },
-          nameInt32: {
-            categorical: false,
-            range: { min: 99, max: 99, nan: 0, ninf: 0, pinf: 0 }
-          },
-          nameCategorical: {
-            categorical: true,
-            categories: expect.arrayContaining([1, false, "0"]),
-            categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]),
-            numCategories: 3
-          }
-        },
-        var: {}
-      })
-    );
-  });
-
-  test("non-finite numbers", () => {
-    const obsAnnotations = [
-      {
-        __index__: 0,
-        name: "n0",
-        nameString: "hi",
-        nameBoolean: false,
-        nameFloat32: 39.3,
-        nameInt32: 99,
-        nameCategorical: 1
-      },
-      {
-        __index__: 1,
-        name: "n1",
-        nameString: "hi",
-        nameBoolean: true,
-        nameFloat32: Number.NEGATIVE_INFINITY,
-        nameInt32: 99,
-        nameCategorical: false
-      },
-      {
-        __index__: 2,
-        name: "n2",
-        nameString: "bye",
-        nameBoolean: true,
-        nameFloat32: Number.NaN,
-        nameInt32: 99,
-        nameCategorical: "0"
-      },
-      {
-        __index__: 3,
-        name: "n2",
-        nameString: "bye",
-        nameBoolean: true,
-        nameFloat32: Number.POSITIVE_INFINITY,
-        nameInt32: 99,
-        nameCategorical: "0"
-      }
-    ];
-    const varAnnotations = [];
-
-    const summary = summarizeAnnotations(
-      schema,
-      obsAnnotations,
-      varAnnotations
-    );
-
-    expect(summary).toMatchObject(
-      expect.objectContaining({
-        obs: {
-          nameString: {
-            categorical: true,
-            categories: expect.arrayContaining(["hi", "bye"]),
-            categoryCounts: new Map([["hi", 2], ["bye", 1]]),
-            numCategories: 2
-          },
-          nameBoolean: {
-            categorical: true,
-            categories: expect.arrayContaining([true, false]),
-            categoryCounts: new Map([[true, 2], [false, 1]]),
-            numCategories: 2
-          },
-          nameFloat32: {
-            categorical: false,
-            range: { min: 39.3, max: 39.3, nan: 1, ninf: 1, pinf: 1 }
-          },
-          nameInt32: {
-            categorical: false,
-            range: { min: 99, max: 99, nan: 0, ninf: 0, pinf: 0 }
-          },
-          nameCategorical: {
-            categorical: true,
-            categories: expect.arrayContaining([1, false, "0"]),
-            categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]),
-            numCategories: 3
-          }
-        },
-        var: {}
-      })
-    );
-  });
-});

client/__tests__/util/stateManager/universe.test.js

@@ -1,13 +1,13 @@
-import _ from "lodash";
 import * as Universe from "../../../src/util/stateManager/universe";
+import * as Dataframe from "../../../src/util/dataframe";
 import * as REST from "./sampleResponses";
 
-describe("createUniverseFromRestV02Response", () => {
+describe("createUniverseFromResponse", () => {
   /*
-  test createUniverseFromRestV02Response - this function converts
+  test createUniverseFromResponse - this function converts
   a set of REST 0.2 responses into a "new" Universe.
 
-  createUniverseFromRestV02Response(
+  createUniverseFromResponse(
     configResponse,
     schemaResponse,
     annotationsObsResponse,
@@ -30,7 +30,7 @@ describe("createUniverseFromRestV02Response", () => {
     create a universe from sample data nad validate its shape & contents
     */
     const { nObs, nVar } = REST.schema.schema.dataframe;
-    const universe = Universe.createUniverseFromRestV02Response(
+    const universe = Universe.createUniverseFromResponse(
       REST.config,
       REST.schema,
       REST.annotationsObs,
@@ -41,27 +41,26 @@ describe("createUniverseFromRestV02Response", () => {
     expect(universe).toBeDefined();
     expect(universe).toMatchObject(
       expect.objectContaining({
-        api: "0.2",
         nObs,
         nVar,
         schema: REST.schema.schema,
-        obsAnnotations: expect.any(Array),
-        varAnnotations: expect.any(Array),
-        obsNameToIndexMap: expect.any(Object),
-        varNameToIndexMap: expect.any(Object),
-        obsLayout: expect.objectContaining({
-          X: expect.any(Float32Array),
-          Y: expect.any(Float32Array)
-        }),
-        varDataCache: expect.any(Object)
+        obsAnnotations: expect.any(Dataframe.Dataframe),
+        varAnnotations: expect.any(Dataframe.Dataframe),
+        obsLayout: expect.any(Dataframe.Dataframe),
+        varData: expect.any(Dataframe.Dataframe)
       })
     );
 
-    expect(universe.obsAnnotations).toHaveLength(nObs);
-    expect(_.keys(universe.obsNameToIndexMap)).toHaveLength(nObs);
-    expect(universe.obsLayout.X).toHaveLength(nObs);
-    expect(universe.obsLayout.Y).toHaveLength(nObs);
-    expect(universe.varAnnotations).toHaveLength(nVar);
-    expect(_.keys(universe.varNameToIndexMap)).toHaveLength(nVar);
+    expect(universe.obsAnnotations.dims).toEqual([
+      nObs,
+      REST.schema.schema.annotations.obs.length
+    ]);
+    expect(universe.obsLayout.dims).toEqual([nObs, 2]);
+    expect(universe.obsLayout.colIndex.keys()).toEqual(["X", "Y"]);
+    expect(universe.varAnnotations.dims).toEqual([
+      nVar,
+      REST.schema.schema.annotations.var.length
+    ]);
+    expect(universe.varData.isEmpty()).toBeTruthy();
   });
 });

client/__tests__/util/stateManager/world.test.js

@@ -1,13 +1,14 @@
 import _ from "lodash";
 import * as Universe from "../../../src/util/stateManager/universe";
 import * as World from "../../../src/util/stateManager/world";
+import * as Dataframe from "../../../src/util/dataframe";
 import Crossfilter from "../../../src/util/typedCrossfilter";
+import { DimTypes } from "../../../src/util/typedCrossfilter/crossfilter";
 import * as REST from "./sampleResponses";
 import {
   obsAnnoDimensionName,
   layoutDimensionName
 } from "../../../src/util/nameCreators";
-import * as kvCache from "../../../src/util/stateManager/keyvalcache";
 
 /*
 Helper - creates universe, world, corssfilter and dimensionMap from
@@ -16,7 +17,7 @@ the default REST test response.
 const defaultBigBang = () => {
   /* create unverse, world, crossfilter and dimensionMap */
   /* create universe */
-  const universe = Universe.createUniverseFromRestV02Response(
+  const universe = Universe.createUniverseFromResponse(
     REST.config,
     REST.schema,
     REST.annotationsObs,
@@ -26,21 +27,21 @@ const defaultBigBang = () => {
   /* create world */
   const world = World.createWorldFromEntireUniverse(universe);
   /* create crossfilter */
-  const crossfilter = Crossfilter(world.obsAnnotations);
-  /* create dimension map */
-  const dimensionMap = World.createObsDimensionMap(crossfilter, world);
+  const crossfilter = World.createObsDimensions(
+    new Crossfilter(world.obsAnnotations),
+    world
+  );
 
   return {
     universe,
     world,
-    crossfilter,
-    dimensionMap
+    crossfilter
   };
 };
 
 describe("createWorldFromEntireUniverse", () => {
   test("create from REST sample", () => {
-    const universe = Universe.createUniverseFromRestV02Response(
+    const universe = Universe.createUniverseFromResponse(
       REST.config,
       REST.schema,
       REST.annotationsObs,
@@ -54,31 +55,13 @@ describe("createWorldFromEntireUniverse", () => {
 
     expect(world).toMatchObject(
       expect.objectContaining({
-        api: "0.2",
         nObs: universe.nObs,
         nVar: universe.nVar,
         schema: universe.schema,
         obsAnnotations: universe.obsAnnotations,
         varAnnotations: universe.varAnnotations,
         obsLayout: universe.obsLayout,
-
-        summary: expect.objectContaining({
-          obs: _(REST.schema.schema.annotations.obs)
-            .filter(v => v.name !== "name")
-            .keyBy("name")
-            .mapValues(() => expect.any(Object))
-            .value(),
-          var: _(REST.schema.schema.annotations.var)
-            .filter(v => v.name !== "name")
-            .keyBy("name")
-            .mapValues(() => expect.any(Object))
-            .value()
-        }),
-
-        varDataCache: expect.any(Object),
-
-        obsIndex: null, // null indicating full universe
-        obsBackIndex: null
+        varData: expect.any(Dataframe.Dataframe)
       })
     );
   });
@@ -89,13 +72,16 @@ describe("createWorldFromCurrentSelection", () => {
     const {
       universe,
       world: originalWorld,
-      crossfilter,
-      dimensionMap
+      crossfilter: originalCrossfilter
     } = defaultBigBang();
 
     /* mock a selection */
-    dimensionMap[obsAnnoDimensionName("field1")].filterRange([0, 5]);
-    dimensionMap[obsAnnoDimensionName("field3")].filterExact(false);
+    let crossfilter = originalCrossfilter
+      .select(obsAnnoDimensionName("field1"), { mode: "range", lo: 0, hi: 5 })
+      .select(obsAnnoDimensionName("field3"), {
+        mode: "exact",
+        values: [false]
+      });
 
     /* create the world from the selection */
     const world = World.createWorldFromCurrentSelection(
@@ -104,58 +90,45 @@ describe("createWorldFromCurrentSelection", () => {
       crossfilter
     );
     expect(world).toBeDefined();
-    expect(world.nObs).toEqual(crossfilter.countFiltered());
+    expect(world.nObs).toEqual(crossfilter.countSelected());
 
     /*
     calculate expected values and match against result
     */
 
     /* matchFilter must match the dimension filters above */
-    const matchFilter = val => val.field1 >= 0 && val.field1 < 5 && !val.field3;
-    const universeIndices = _()
-      .range(universe.nObs)
-      .filter(idx => matchFilter(universe.obsAnnotations[idx]))
-      .value();
-
-    const expected = {
-      nObs: universeIndices.length,
-      obsAnnotations: _.map(universeIndices, i => universe.obsAnnotations[i]),
-      obsLayout: {
-        X: new Float32Array(
-          _.map(universeIndices, i => universe.obsLayout.X[i])
-        ),
-        Y: new Float32Array(
-          _.map(universeIndices, i => universe.obsLayout.Y[i])
-        )
-      },
-      obsBackIndex: _.transform(
-        universeIndices,
-        (result, univIdx, worldIdx) => {
-          result[univIdx] = worldIdx;
-        },
-        new Uint32Array(universe.nObs).fill(-1)
-      ),
-      obsIndex: new Uint32Array(universeIndices)
+    const matchFilter = (df, row) => {
+      const field1 = df.at(row, "field1");
+      const field3 = df.at(row, "field3");
+      return field1 >= 0 && field1 < 5 && !field3;
     };
+    const matchingIndices = _()
+      .range(universe.nObs)
+      .filter(idx => matchFilter(universe.obsAnnotations, idx))
+      .value();
 
     expect(world).toMatchObject(
       expect.objectContaining({
-        api: "0.2",
-        nObs: expected.nObs,
+        nObs: matchingIndices.length,
         nVar: universe.nVar,
         schema: universe.schema,
-        obsAnnotations: expected.obsAnnotations,
+        obsAnnotations: expect.any(Dataframe.Dataframe),
         varAnnotations: universe.varAnnotations,
-        obsLayout: expected.obsLayout,
-        summary: {
-          obs: expect.any(Object) /* we could do better! */,
-          var: expect.any(Object) /* we could do better! */
-        },
-        varDataCache: expect.any(Object),
-        obsIndex: expected.obsIndex,
-        obsBackIndex: expected.obsBackIndex
+        obsLayout: expect.any(Dataframe.Dataframe),
+        varData: expect.any(Dataframe.Dataframe)
       })
     );
+
+    expect(world.obsAnnotations.rowIndex.keys()).toEqual(
+      new Int32Array(matchingIndices)
+    );
+    expect(world.obsAnnotations.colIndex.keys()).toEqual(
+      universe.obsAnnotations.colIndex.keys()
+    );
+    expect(world.obsLayout.rowIndex.keys()).toEqual(
+      new Int32Array(matchingIndices)
+    );
+    expect(world.obsLayout.colIndex.keys()).toEqual(["X", "Y"]);
   });
 });
 
@@ -167,87 +140,32 @@ describe("createObsDimensionMap", () => {
     - check that dimension typing is sane
     */
 
-    const { dimensionMap } = defaultBigBang();
+    const { crossfilter } = defaultBigBang();
     const annotationNames = _.map(
       REST.schema.schema.annotations.obs,
       c => c.name
     );
     const schemaByObsName = _.keyBy(REST.schema.schema.annotations.obs, "name");
-    expect(dimensionMap).toBeDefined();
+    expect(crossfilter).toBeDefined();
     annotationNames.forEach(name => {
-      const dim = dimensionMap[obsAnnoDimensionName(name)];
+      const dim = crossfilter.dimensions[obsAnnoDimensionName(name)];
       if (name === "name") {
         expect(dim).toBeUndefined();
       } else {
         const { type } = schemaByObsName[name];
         if (type === "string" || type === "boolean" || type === "categorical") {
-          expect(dim).toBeInstanceOf(Crossfilter.EnumDimension);
+          expect(dim.dim).toBeInstanceOf(DimTypes.enum);
         } else {
-          expect(dim).toBeInstanceOf(Crossfilter.ScalarDimension);
+          expect(dim.dim).toBeInstanceOf(DimTypes.scalar);
         }
       }
     });
-    expect(dimensionMap[layoutDimensionName("XY")]).toBeInstanceOf(
-      Crossfilter.SpatialDimension
-    );
+    expect(
+      crossfilter.dimensions[layoutDimensionName("XY")].dim
+    ).toBeInstanceOf(DimTypes.spatial);
   });
 });
 
-describe("subsetVarData", () => {
-  test("when world eq universe", () => {
-    const { universe, world } = defaultBigBang();
-    /* create a mock varData array for subsetting */
-    const sourceVarData = new Float32Array(universe.nObs);
-
-    /* expect literally the same object back */
-    const result = World.subsetVarData(world, universe, sourceVarData);
-    expect(result).toBe(sourceVarData);
-  });
-
-  test("when world neq universe", () => {
-    const { universe, world, crossfilter, dimensionMap } = defaultBigBang();
-    /* create a mock varData array for subsetting */
-    const sourceVarData = Float32Array.from(_.range(universe.nObs));
-
-    /* mock a selection */
-    dimensionMap[obsAnnoDimensionName("field1")].filterRange([0, 5]);
-    dimensionMap[obsAnnoDimensionName("field3")].filterExact(false);
-
-    /* create the world from the selection */
-    const newWorld = World.createWorldFromCurrentSelection(
-      universe,
-      world,
-      crossfilter
-    );
-    expect(newWorld.obsIndex).toMatchObject(new Uint32Array([0, 2]));
-
-    /* expect a subset */
-    const result = World.subsetVarData(newWorld, universe, sourceVarData);
-    expect(result).not.toBe(sourceVarData);
-    expect(result).toHaveLength(newWorld.nObs);
-    /* check that we have expected source var content */
-    expect(result).toMatchObject(new Float32Array([0, 2]));
-  });
-});
-
-describe("createVarDimension", () => {
-  /* create default universe */
-  const { world, crossfilter } = defaultBigBang();
-  /* create a mock var data cache */
-  const varDataCache = kvCache.set(
-    kvCache.create(),
-    "GENE",
-    Float32Array.from(_.range(world.nObs))
-  );
-  const result = World.createVarDimension(
-    world,
-    varDataCache,
-    crossfilter,
-    "GENE"
-  );
-  expect(result).toBeInstanceOf(Crossfilter.ScalarDimension);
-});
-
 describe("worldEqUniverse", () => {
   const { universe, world } = defaultBigBang();
   const result = World.worldEqUniverse(world, universe);

client/__tests__/util/stateManager/worldUtil.test.js

@@ -2,16 +2,27 @@ import {
   countCategoryValues2D,
   clearCaches
 } from "../../../src/util/stateManager/worldUtil";
+import * as Dataframe from "../../../src/util/dataframe";
 
 describe("WorldUtil cache management", () => {
   test("empty", () => {
-    const count = countCategoryValues2D("a", "b", []);
+    const count = countCategoryValues2D(
+      "a",
+      "b",
+      new Dataframe.Dataframe([0, 0], [])
+    );
     expect(count).toMatchObject(new Map());
+    expect(count.size).toBe(0);
   });
 
   test("simple couts", () => {
-    const rows = [{ a: 0, b: false }, { a: 0, b: true }, { a: 1, b: false }];
-    const count = countCategoryValues2D("a", "b", rows);
+    const df = new Dataframe.Dataframe(
+      [3, 2],
+      [[0, 0, 1], [false, true, false]],
+      null,
+      new Dataframe.KeyIndex(["a", "b"])
+    );
+    const count = countCategoryValues2D("a", "b", df);
     expect(count).toMatchObject(
       new Map([
         [0, new Map([[true, 1], [false, 1]])],
@@ -22,16 +33,22 @@ describe("WorldUtil cache management", () => {
 
   test("memo cache clear", () => {
     clearCaches();
-    const row1 = [];
-    const row2 = [{ a: 0, b: false }, { a: 0, b: true }, { a: 1, b: false }];
-    const count1 = countCategoryValues2D("a", "b", row1);
-    const count2 = countCategoryValues2D("a", "b", row1);
-    const count3 = countCategoryValues2D("a", "b", []);
-    const count4 = countCategoryValues2D("a", "b", row2);
+    const df1 = new Dataframe.Dataframe([0, 0], []);
+    const df2 = new Dataframe.Dataframe(
+      [3, 2],
+      [[0, 0, 1], [false, true, false]],
+      null,
+      new Dataframe.KeyIndex(["a", "b"])
+    );
+
+    const count1 = countCategoryValues2D("a", "b", df1);
+    const count2 = countCategoryValues2D("a", "b", df1);
+    const count3 = countCategoryValues2D("a", "b", df1.clone());
+    const count4 = countCategoryValues2D("a", "b", df2);
 
     clearCaches();
-    const count10 = countCategoryValues2D("a", "b", row1);
-    const count11 = countCategoryValues2D("a", "b", row2);
+    const count10 = countCategoryValues2D("a", "b", df1);
+    const count11 = countCategoryValues2D("a", "b", df2);
 
     expect(count1).toEqual(count2);
     expect(count1).toEqual(count3);

client/__tests__/util/typedCrossfilter/bitArray.test.js

@@ -118,16 +118,16 @@ describe("selectionCount", () => {
     const dim2 = ba.allocDimension();
     expect(dim2).toBeDefined();
 
-    expect(ba.selectionCount).toEqual(0);
+    expect(ba.selectionCount()).toEqual(0);
     ba.selectAll(dim1);
-    expect(ba.selectionCount).toEqual(0);
+    expect(ba.selectionCount()).toEqual(0);
     ba.selectAll(dim2);
-    expect(ba.selectionCount).toEqual(defaultTestLength);
+    expect(ba.selectionCount()).toEqual(defaultTestLength);
 
     for (let i = 0; i < defaultTestLength; i += 1) {
       ba.deselectOne(dim1, i);
-      expect(ba.selectionCount).toEqual(defaultTestLength - i - 1);
-      expect(ba.selectionCount).toEqual(ba.countAllOnes());
+      expect(ba.selectionCount()).toEqual(defaultTestLength - i - 1);
+      expect(ba.selectionCount()).toEqual(ba.countAllOnes());
     }
 
     ba.freeDimension(dim1);

client/__tests__/util/typedCrossfilter/crossfilter.test.js

@@ -0,0 +1,330 @@
+import _ from "lodash";
+import { polygonContains } from "d3";
+
+import Crossfilter from "../../../src/util/typedCrossfilter";
+
+const someData = [
+  {
+    date: "2011-11-14T16:17:54Z",
+    quantity: 2,
+    total: 190,
+    tip: 100,
+    type: "tab",
+    productIDs: ["001"],
+    coords: [0, 0]
+  },
+  {
+    date: "2011-11-14T16:20:19Z",
+    quantity: 2,
+    total: 190,
+    tip: 100,
+    type: "tab",
+    productIDs: ["001", "005"],
+    coords: [0.4, 0.4]
+  },
+  {
+    date: "2011-11-14T16:28:54Z",
+    quantity: 1,
+    total: 300,
+    tip: 200,
+    type: "visa",
+    productIDs: ["004", "005"],
+    coords: [0.3, 0.1]
+  },
+  {
+    date: "2011-11-14T16:30:43Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["001", "002"],
+    coords: [0.392, 0.1]
+  },
+  {
+    date: "2011-11-14T16:48:46Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["005"],
+    coords: [0.7, 0.0482]
+  },
+  {
+    date: "2011-11-14T16:53:41Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["001", "004", "005"],
+    coords: [0.9999, 1.0]
+  },
+  {
+    date: "2011-11-14T16:54:06Z",
+    quantity: 1,
+    total: 100,
+    tip: 0,
+    type: "cash",
+    productIDs: ["001", "002", "003", "004", "005"],
+    coords: [0.384, 0.6938]
+  },
+  {
+    date: "2011-11-14T16:58:03Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["001"],
+    coords: [0.4822, 0.482]
+  },
+  {
+    date: "2011-11-14T17:07:21Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["004", "005"],
+    coords: [0.2234, 0]
+  },
+  {
+    date: "2011-11-14T17:22:59Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["001", "002", "004", "005"],
+    coords: [0.382, 0.38485]
+  },
+  {
+    date: "2011-11-14T17:25:45Z",
+    quantity: 2,
+    total: 200,
+    tip: 0,
+    type: "cash",
+    productIDs: ["002"],
+    coords: [0.998, 0.8472]
+  },
+  {
+    date: "2011-11-14T17:29:52Z",
+    quantity: 1,
+    total: 200,
+    tip: 100,
+    type: "visa",
+    productIDs: ["004"],
+    coords: [0.8273, 0.3384]
+  }
+];
+
+let payments = null;
+beforeEach(() => {
+  payments = new Crossfilter(someData);
+});
+
+describe("ImmutableTypedCrossfilter", () => {
+  test("create crossfilter", () => {
+    expect(payments).toBeDefined();
+    expect(payments.size()).toEqual(someData.length);
+    expect(payments.all()).toEqual(someData);
+
+    const p = payments
+      .addDimension("quantity", "scalar", (i, d) => d[i].quantity, Int32Array)
+      .select("quantity", { mode: "all" });
+    expect(p).toBeDefined();
+    expect(p.all()).toEqual(someData);
+    expect(p.size()).toEqual(someData.length);
+    expect(p.isElementSelected(0)).toBeTruthy();
+    expect(p.countSelected()).toEqual(someData.length);
+    expect(p.allSelected()).toEqual(someData);
+  });
+
+  test("immutability", () => {
+    /*
+    the following should return a new crossfilter:
+    - addDimension()
+    - delDimension()
+    - select
+    */
+    const p2 = payments.addDimension(
+      "quantity",
+      "scalar",
+      (i, data) => data[i].quantity,
+      Int32Array
+    );
+
+    expect(payments).not.toBe(p2);
+    const p3 = p2.select("quantity", { mode: "all" });
+    expect(p3).not.toBe(p2);
+
+    const p4 = p3.delDimension("quantity");
+    expect(p4).not.toBe(p3);
+  });
+
+  test("select all and none", () => {
+    let p = payments
+      .addDimension("quantity", "scalar", (i, d) => d[i].quantity, Int32Array)
+      .addDimension("tip", "scalar", (i, d) => d[i].tip, Float32Array)
+      .addDimension("total", "scalar", (i, d) => d[i].total, Float32Array)
+      .addDimension("type", "enum", (i, d) => d[i].type);
+    expect(p).toBeDefined();
+
+    /* expect all records to be selected - default init state */
+    expect(p.allSelected()).toEqual(someData);
+    expect(p.countSelected()).toEqual(someData.length);
+    expect(p.allSelectedMask()).toEqual(
+      new Uint8Array(someData.length).fill(1)
+    );
+    expect(p.fillByIsSelected(new Uint8Array(someData.length), 99, 0)).toEqual(
+      new Uint8Array(someData.length).fill(99)
+    );
+    for (let i = 0; i < someData.length; i += 1) {
+      expect(p.isElementSelected(i)).toBeTruthy();
+    }
+
+    /* expect a selectAll on one dimension to change nothing */
+    p = p.select("tip", { mode: "all" });
+    expect(p.allSelected()).toEqual(someData);
+
+    /* ditto */
+    p = p.select("quantity", { mode: "all" });
+    expect(p.allSelected()).toEqual(someData);
+
+    /* select none on one dimension */
+    p = p.select("type", { mode: "none" });
+    expect(p.allSelected()).toEqual([]);
+    expect(p.countSelected()).toEqual(0);
+    expect(p.allSelectedMask()).toEqual(
+      new Uint8Array(someData.length).fill(0)
+    );
+    expect(p.fillByIsSelected(new Uint8Array(someData.length), 99, 0)).toEqual(
+      new Uint8Array(someData.length).fill(0)
+    );
+    for (let i = 0; i < someData.length; i += 1) {
+      expect(p.isElementSelected(i)).toBeFalsy();
+    }
+
+    p = p.select("quantity", { mode: "none" });
+    expect(p.allSelected()).toEqual([]);
+
+    // invert the first none; should have no effect because type is
+    // still not filtered.
+    p = p.select("quantity", { mode: "all" });
+    expect(p.allSelected()).toEqual([]);
+
+    /* select all of type; should select all records */
+    p = p.select("type", { mode: "all" });
+    expect(p.allSelected()).toEqual(someData);
+  });
+
+  describe("scalar dimension", () => {
+    let p;
+    beforeEach(() => {
+      p = payments
+        .addDimension("quantity", "scalar", (i, d) => d[i].quantity, Int32Array)
+        .addDimension("tip", "scalar", (i, d) => d[i].tip, Float32Array)
+        .select("tip", { mode: "all" });
+    });
+
+    /*
+    select modes:  all, none, exact, range
+    */
+    test("all", () => {
+      expect(p.select("quantity", { mode: "all" }).countSelected()).toEqual(
+        someData.length
+      );
+    });
+    test("none", () => {
+      expect(p.select("quantity", { mode: "none" }).countSelected()).toEqual(0);
+    });
+    test.each([[[]], [[2]], [[2, 1]], [[9, 82]], [[0, 1]]])("exact: %p", v =>
+      expect(
+        p.select("quantity", { mode: "exact", values: v }).countSelected()
+      ).toEqual(_.filter(someData, d => v.includes(d.quantity)).length)
+    );
+    test.each([[0, 1], [1, 2], [0, 99], [99, 100000]])("range %p", (lo, hi) =>
+      expect(
+        p.select("quantity", { mode: "range", lo, hi }).countSelected()
+      ).toEqual(
+        _.filter(someData, d => d.quantity >= lo && d.quantity < hi).length
+      )
+    );
+    test("bad mode", () => {
+      expect(() => p.select("type", { mode: "bad mode" })).toThrow(Error);
+    });
+  });
+
+  describe("enum dimension", () => {
+    let p;
+    beforeEach(() => {
+      p = payments.addDimension("type", "enum", (i, d) => d[i].type);
+    });
+
+    test("all", () => {
+      expect(p.select("type", { mode: "all" }).countSelected()).toEqual(
+        someData.length
+      );
+    });
+    test("none", () => {
+      expect(p.select("type", { mode: "none" }).countSelected()).toEqual(0);
+    });
+    test.each([
+      [[]],
+      [["tab"]],
+      [["visa"]],
+      [["visa", "tab"]],
+      [["cash", "tab", "visa"]]
+    ])("exact: %p", v =>
+      expect(
+        p.select("type", { mode: "exact", values: v }).countSelected()
+      ).toEqual(_.filter(someData, d => v.includes(d.type)).length)
+    );
+    test("range", () => {
+      expect(() => p.select("type", { mode: "range", lo: 0, hi: 9 })).toThrow(
+        Error
+      );
+    });
+    test("bad mode", () => {
+      expect(() => p.select("type", { mode: "bad mode" })).toThrow(Error);
+    });
+  });
+
+  describe("spatial dimension", () => {
+    let p;
+    beforeEach(() => {
+      const X = someData.map(r => r.coords[0]);
+      const Y = someData.map(r => r.coords[1]);
+      p = payments.addDimension("coords", "spatial", X, Y);
+    });
+
+    test("all", () => {
+      expect(p.select("coords", { mode: "all" }).countSelected()).toEqual(
+        someData.length
+      );
+    });
+    test("none", () => {
+      expect(p.select("coords", { mode: "none" }).countSelected()).toEqual(0);
+    });
+    test.each([[0, 0, 1, 1], [0, 0, 0.5, 0.5], [0.5, 0.5, 1, 1]])(
+      "within-rect %d %d %d %d",
+      (x0, y0, x1, y1) => {
+        expect(
+          p
+            .select("coords", { mode: "within-rect", x0, y0, x1, y1 })
+            .allSelected()
+        ).toEqual(
+          _.filter(someData, d => {
+            const [x, y] = d.coords;
+            return x0 <= x && x < x1 && y0 <= y && y < y1;
+          })
+        );
+      }
+    );
+
+    test.each([
+      [[[0, 0], [0, 1], [1, 1], [1, 0]]],
+      [[[0, 0], [0, 0.5], [0.5, 0.5], [0.5, 0]]]
+    ])("within-polygon %p", polygon => {
+      expect(
+        p.select("coords", { mode: "within-polygon", polygon }).allSelected()
+      ).toEqual(_.filter(someData, d => polygonContains(polygon, d.coords)));
+    });
+  });
+});

client/__tests__/util/typedCrossfilter/typedCrossfilter.test.js

@@ -1,560 +0,0 @@
-// jshint esversion: 6
-import _ from "lodash";
-import crossfilter from "../../../src/util/typedCrossfilter";
-
-const someData = [
-  {
-    date: "2011-11-14T16:17:54Z",
-    quantity: 2,
-    total: 190,
-    tip: 100,
-    type: "tab",
-    productIDs: ["001"]
-  },
-  {
-    date: "2011-11-14T16:20:19Z",
-    quantity: 2,
-    total: 190,
-    tip: 100,
-    type: "tab",
-    productIDs: ["001", "005"]
-  },
-  {
-    date: "2011-11-14T16:28:54Z",
-    quantity: 1,
-    total: 300,
-    tip: 200,
-    type: "visa",
-    productIDs: ["004", "005"]
-  },
-  {
-    date: "2011-11-14T16:30:43Z",
-    quantity: 2,
-    total: 90,
-    tip: 0,
-    type: "tab",
-    productIDs: ["001", "002"]
-  },
-  {
-    date: "2011-11-14T16:48:46Z",
-    quantity: 2,
-    total: 90,
-    tip: 0,
-    type: "tab",
-    productIDs: ["005"]
-  },
-  {
-    date: "2011-11-14T16:53:41Z",
-    quantity: 2,
-    total: 90,
-    tip: 0,
-    type: "tab",
-    productIDs: ["001", "004", "005"]
-  },
-  {
-    date: "2011-11-14T16:54:06Z",
-    quantity: 1,
-    total: 100,
-    tip: 0,
-    type: "cash",
-    productIDs: ["001", "002", "003", "004", "005"]
-  },
-  {
-    date: "2011-11-14T16:58:03Z",
-    quantity: 2,
-    total: 90,
-    tip: 0,
-    type: "tab",
-    productIDs: ["001"]
-  },
-  {
-    date: "2011-11-14T17:07:21Z",
-    quantity: 2,
-    total: 90,
-    tip: 0,
-    type: "tab",
-    productIDs: ["004", "005"]
-  },
-  {
-    date: "2011-11-14T17:22:59Z",
-    quantity: 2,
-    total: 90,
-    tip: 0,
-    type: "tab",
-    productIDs: ["001", "002", "004", "005"]
-  },
-  {
-    date: "2011-11-14T17:25:45Z",
-    quantity: 2,
-    total: 200,
-    tip: 0,
-    type: "cash",
-    productIDs: ["002"]
-  },
-  {
-    date: "2011-11-14T17:29:52Z",
-    quantity: 1,
-    total: 200,
-    tip: 100,
-    type: "visa",
-    productIDs: ["004"]
-  }
-];
-
-function groupReduce(data, valueMap, valueReduce, valueInit) {
-  return _.reduce(
-    data,
-    (acc, value) => {
-      const k = valueMap(value);
-      let r = _.find(acc, o => o.key === k);
-      if (!r) {
-        r = { key: k, value: valueInit() };
-        acc.push(r);
-      }
-      r.value = valueReduce(r.value, value);
-      return acc;
-    },
-    []
-  ).sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
-}
-
-function groupCount(data, map) {
-  return groupReduce(data, map, (p, v) => p + 1, () => 0);
-}
-
-function groupSum(data, map) {
-  return groupReduce(data, map, (p, v) => (p += map(v)), () => 0);
-}
-
-var payments = null;
-beforeEach(() => {
-  payments = crossfilter(someData);
-});
-
-describe("typedCrossfilter", () => {
-  test("alloc and free", () => {
-    expect(payments).toBeDefined();
-    expect(payments.size()).toEqual(someData.length);
-    expect(payments.all()).toEqual(someData);
-
-    const quantity = payments.dimension(
-      crossfilter.ScalarDimension,
-      r => r.quantity,
-      Int32Array
-    );
-    expect(quantity).toBeDefined();
-    expect(quantity.id()).toBeDefined();
-
-    quantity.dispose();
-    expect(payments.size()).toEqual(someData.length);
-    expect(payments.all()).toEqual(someData);
-  });
-
-  test("filterAll and filterNone", () => {
-    expect(payments).toBeDefined();
-    const quantity = payments.dimension(
-      crossfilter.ScalarDimension,
-      r => r.quantity,
-      Int32Array
-    );
-    const tip = payments.dimension(
-      crossfilter.ScalarDimension,
-      r => r.tip,
-      Float32Array
-    );
-    const total = payments.dimension(
-      crossfilter.ScalarDimension,
-      r => r.total,
-      Float32Array
-    );
-    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
-
-    expect(quantity).toBeDefined();
-    expect(tip).toBeDefined();
-    expect(total).toBeDefined();
-    expect(type).toBeDefined();
-
-    // initially, all should be filtered
-    expect(payments.allFiltered()).toHaveLength(payments.size());
-    expect(payments.allFiltered()).toEqual(payments.all());
-    expect(payments.countFiltered()).toEqual(someData.length);
-
-    // filterAll
-    tip.filterAll(); // should change nothing
-    expect(payments.allFiltered()).toEqual(payments.all());
-    expect(payments.countFiltered()).toEqual(someData.length);
-
-    // ditto
-    total.filterAll();
-    expect(payments.allFiltered()).toEqual(payments.all());
-    expect(payments.countFiltered()).toEqual(someData.length);
-
-    // filterNone
-    type.filterNone();
-    expect(payments.allFiltered()).toEqual([]);
-    expect(payments.countFiltered()).toEqual(0);
-
-    quantity.filterNone();
-    expect(payments.allFiltered()).toEqual([]);
-    expect(payments.countFiltered()).toEqual(0);
-
-    // invert the first none; should have no effect because type is
-    // still not filtered
-    quantity.filterAll();
-    expect(payments.allFiltered()).toEqual([]);
-    expect(payments.countFiltered()).toEqual(0);
-
-    // filter all of type; should select all
-    type.filterAll();
-    expect(payments.allFiltered()).toEqual(payments.all());
-    expect(payments.countFiltered()).toEqual(payments.size());
-  });
-
-  test("filterExact", () => {
-    expect(payments).toBeDefined();
-    const quantity = payments.dimension(
-      crossfilter.ScalarDimension,
-      r => r.quantity,
-      Int32Array
-    );
-    const tip = payments.dimension(
-      crossfilter.ScalarDimension,
-      r => r.tip,
-      Float32Array
-    );
-    const total = payments.dimension(
-      crossfilter.ScalarDimension,
-      r => r.total,
-      Float32Array
-    );
-    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
-
-    quantity.filterExact(1);
-    expect(payments.countFiltered()).toEqual(
-      _.countBy(someData, "quantity")[1]
-    );
-    expect(payments.allFiltered()).toEqual(_.filter(someData, { quantity: 1 }));
-
-    tip.filterExact(0);
-    expect(payments.allFiltered()).toEqual(
-      _.filter(someData, { tip: 0, quantity: 1 })
-    );
-
-    type.filterExact("cash");
-    expect(payments.allFiltered()).toEqual(
-      _.filter(someData, { tip: 0, quantity: 1, type: "cash" })
-    );
-  });
-
-  test("filterRange", () => {
-    expect(payments).toBeDefined();
-    const quantity = payments.dimension(
-      crossfilter.ScalarDimension,
-      r => r.quantity,
-      Int32Array
-    );
-    const tip = payments.dimension(
-      crossfilter.ScalarDimension,
-      r => r.tip,
-      Float32Array
-    );
-    const total = payments.dimension(
-      crossfilter.ScalarDimension,
-      r => r.total,
-      Float32Array
-    );
-    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
-
-    tip.filterRange([0, 91]);
-    expect(payments.allFiltered()).toEqual(
-      _(someData)
-        .filter(r => r.tip >= 0 && r.tip < 91)
-        .value()
-    );
-
-    tip.filterRange([0, 90]);
-    expect(payments.allFiltered()).toEqual(
-      _(someData)
-        .filter(r => r.tip >= 0 && r.tip < 90)
-        .value()
-    );
-
-    tip.filterRange([1, 90]);
-    expect(payments.allFiltered()).toEqual(
-      _(someData)
-        .filter(r => r.tip >= 1 && r.tip < 91)
-        .value()
-    );
-  });
-
-  test("filterEnum", () => {
-    expect(payments).toBeDefined();
-    const quantity = payments.dimension(
-      crossfilter.ScalarDimension,
-      r => r.quantity,
-      Int32Array
-    );
-    const tip = payments.dimension(
-      crossfilter.ScalarDimension,
-      r => r.tip,
-      Float32Array
-    );
-    const total = payments.dimension(
-      crossfilter.ScalarDimension,
-      r => r.total,
-      Float32Array
-    );
-    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
-
-    type.filterEnum(["tab", "cash"]);
-    expect(payments.allFiltered()).toEqual(
-      _(someData)
-        .filter(r => r.type === "cash" || r.type === "tab")
-        .value()
-    );
-
-    tip.filterEnum([0, 100]);
-    expect(payments.allFiltered()).toEqual(
-      _(someData)
-        .filter(r => r.type === "cash" || r.type === "tab")
-        .filter(r => r.tip === 0 || r.tip === 100)
-        .value()
-    );
-  });
-
-  test("more than 32 dimensions", () => {
-    expect(payments).toBeDefined();
-    const quantity = payments.dimension(
-      crossfilter.ScalarDimension,
-      r => r.quantity,
-      Int32Array
-    );
-    const tip = payments.dimension(
-      crossfilter.ScalarDimension,
-      r => r.tip,
-      Float32Array
-    );
-    const total = payments.dimension(
-      crossfilter.ScalarDimension,
-      r => r.total,
-      Float32Array
-    );
-    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
-
-    // Create a bunch of fake dimensions to ensure we can handle > 32
-    let dimMap = {};
-    for (let i = 0; i < 65; i++) {
-      dimMap[i] = payments.dimension(
-        crossfilter.ScalarDimension,
-        r => Math.random(),
-        Float32Array
-      );
-      expect(dimMap[i]).toBeDefined();
-      expect(dimMap[i].id()).toBeDefined();
-    }
-
-    // everything should start as selected/filtered
-    expect(payments.countFiltered()).toEqual(someData.length);
-
-    dimMap[0].filterAll();
-    dimMap[64].filterAll();
-    expect(payments.countFiltered()).toEqual(someData.length);
-
-    dimMap[33].filterNone();
-    expect(payments.allFiltered()).toEqual([]);
-
-    dimMap[33].filterAll();
-    expect(payments.allFiltered()).toEqual(someData);
-  });
-
-  test("group, default mapping, default reducer, no filter", () => {
-    expect(payments).toBeDefined();
-
-    const quantity = payments.dimension(
-      crossfilter.ScalarDimension,
-      r => r.quantity,
-      Int32Array
-    );
-    const tip = payments.dimension(
-      crossfilter.ScalarDimension,
-      r => r.tip,
-      Int32Array
-    );
-    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
-    const total = payments.dimension(
-      crossfilter.ScalarDimension,
-      r => r.total,
-      Int32Array
-    );
-
-    _.each(
-      {
-        tip: tip.group(r => r),
-        type: type.group(),
-        total: total.group(),
-        quantity: quantity.group()
-      },
-      (grp, k) => {
-        const whatWeExpect = groupCount(someData, v => v[k]);
-        expect(grp.all()).toEqual(whatWeExpect);
-        expect(grp.size()).toEqual(whatWeExpect.length);
-        expect(grp.dispose()).toEqual(grp);
-      }
-    );
-  });
-
-  test("group, custom map, default reducer, no filters", () => {
-    expect(payments).toBeDefined();
-
-    // custom mapping in groups only works for scalar types.  Enums do not
-    // currently implement it.
-
-    const tip = payments.dimension(
-      crossfilter.ScalarDimension,
-      r => r.tip,
-      Int32Array
-    );
-    const totalX10 = payments.dimension(
-      crossfilter.ScalarDimension,
-      r => r.total * 10,
-      Int32Array
-    );
-    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
-
-    const paymentsByTip_A = tip.group();
-    const paymentsByTip_B = tip.group(r => 10 * r);
-    const paymentsByType = type.group(); // identity only
-    const paymentsByTotalX10_A = totalX10.group();
-    const paymentsByTotalX10_B = totalX10.group(r => r / 10);
-
-    expect(paymentsByTip_A.all()).toEqual(groupCount(someData, v => v.tip));
-    expect(paymentsByTip_B.all()).toEqual(
-      groupCount(someData, v => 10 * v.tip)
-    );
-    expect(paymentsByType.all()).toEqual(groupCount(someData, v => v.type));
-    expect(paymentsByTotalX10_A.all()).toEqual(
-      groupCount(someData, v => 10 * v.total)
-    );
-    expect(paymentsByTotalX10_B.all()).toEqual(
-      groupCount(someData, v => (10 * v.total) / 10)
-    );
-
-    for (let i of [
-      paymentsByTip_A,
-      paymentsByTip_B,
-      paymentsByType,
-      paymentsByTotalX10_A,
-      paymentsByTotalX10_B,
-      tip,
-      totalX10,
-      type
-    ]) {
-      expect(i.dispose()).toEqual(i);
-    }
-  });
-
-  test("group, default map, custom reducer, no filters", () => {
-    expect(payments).toBeDefined();
-
-    const total = payments.dimension(
-      crossfilter.ScalarDimension,
-      r => r.total,
-      Float32Array
-    );
-    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
-
-    const paymentsByTotal = total.group();
-    const paymentsByType = type.group();
-
-    // reduceCount
-    expect(paymentsByTotal.reduceCount()).toEqual(paymentsByTotal);
-    expect(paymentsByTotal.all()).toEqual(groupCount(someData, v => v.total));
-
-    // reduceSum
-    expect(paymentsByTotal.reduceSum(v => v.total)).toEqual(paymentsByTotal);
-    expect(paymentsByTotal.all()).toEqual(groupSum(someData, v => v.total));
-
-    // use custom reducers (my reducers) - count by three, init 1
-    expect(
-      paymentsByTotal.reduce((p, v) => (p += 3), (p, v) => (p -= 3), () => 1)
-    ).toEqual(paymentsByTotal);
-    expect(paymentsByTotal.all()).toEqual(
-      groupReduce(someData, v => v.total, (p, v) => p + 3, () => 1)
-    );
-
-    for (let i of [paymentsByTotal, paymentsByType, type]) {
-      expect(i.dispose()).toEqual(i);
-    }
-  });
-
-  test("group, default map, default reducer, filters", () => {
-    // From the docs:
-    // Note: a grouping intersects the crossfilter's current filters, except for the
-    // associated dimension's filter. Thus, group methods consider only records that
-    // satisfy every filter except this dimension's filter. So, if the crossfilter of
-    // payments is filtered by type and total, then group by total only observes the
-    // filter by type.
-
-    expect(payments).toBeDefined();
-
-    const tip = payments.dimension(
-      crossfilter.ScalarDimension,
-      r => r.tip,
-      Int32Array
-    );
-    const total = payments.dimension(
-      crossfilter.ScalarDimension,
-      r => r.total,
-      Int32Array
-    );
-    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
-
-    const paymentsByTip = tip.group();
-    const paymentsByTotal = total.group();
-    const paymentsByType = type.group();
-
-    // 1. confirm that changing the filter on a dimension does NOT change that
-    // dimensions groups.
-    {
-      tip.filterAll(), total.filterAll(), type.filterAll();
-      let before = _.cloneDeep(paymentsByTip.all());
-      tip.filterExact(0);
-      expect(paymentsByTip.all()).toEqual(before);
-    }
-
-    // 2. confirm that changing a filter on a different dimension DOES change
-    // all other groups.
-    {
-      tip.filterAll(), total.filterAll(), type.filterAll();
-      const before = _.cloneDeep([paymentsByTotal.all(), paymentsByType.all()]);
-      tip.filterExact(0);
-      const after = [paymentsByTotal.all(), paymentsByType.all()];
-      expect(after).not.toEqual(before);
-      expect(after).toEqual([
-        groupReduce(
-          someData,
-          v => v.total,
-          (p, v) => (v.tip !== 0 ? p : p + 1),
-          () => 0
-        ),
-        groupReduce(
-          someData,
-          v => v.type,
-          (p, v) => (v.tip !== 0 ? p : p + 1),
-          () => 0
-        )
-      ]);
-    }
-
-    for (let i of [
-      paymentsByTip,
-      paymentsByTotal,
-      paymentsByType,
-      tip,
-      total,
-      type
-    ]) {
-      expect(i.dispose()).toEqual(i);
-    }
-  });
-});

client/configuration/babel/babel.dev.js

@@ -9,6 +9,8 @@ module.exports = {
     "@babel/plugin-proposal-function-bind",
     "@babel/plugin-proposal-class-properties",
     ["@babel/plugin-proposal-decorators", { legacy: true }],
-    "@babel/plugin-proposal-export-namespace-from"
+    "@babel/plugin-proposal-export-namespace-from",
+    "@babel/plugin-proposal-optional-chaining",
+    "@babel/plugin-proposal-nullish-coalescing-operator"
   ]
 };

client/configuration/babel/babel.prod.js

@@ -10,6 +10,8 @@ module.exports = {
     ["@babel/plugin-proposal-decorators", { legacy: true }],
     "@babel/plugin-proposal-export-namespace-from",
     "@babel/plugin-transform-react-constant-elements",
-    "@babel/plugin-transform-runtime"
+    "@babel/plugin-transform-runtime",
+    "@babel/plugin-proposal-optional-chaining",
+    "@babel/plugin-proposal-nullish-coalescing-operator"
   ]
 };

client/configuration/webpack/webpack.config.dev.js

@@ -77,6 +77,9 @@ module.exports = {
       template: path.resolve("index.html"),
       favicon: path.resolve("favicon.png")
     }),
-    new webpack.NoEmitOnErrorsPlugin()
+    new webpack.NoEmitOnErrorsPlugin(),
+    new webpack.DefinePlugin({
+      __REACT_DEVTOOLS_GLOBAL_HOOK__: "({ isDisabled: true })"
+    })
   ]
 };

client/package.json

@@ -1,17 +1,21 @@
 {
   "name": "cellxgene",
-  "version": "0.6.1",
+  "version": "0.8.0",
   "license": "MIT",
   "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
   "repository": "https://github.com/chanzuckerberg/cellxgene",
   "scripts": {
     "backend-dev": "python3.6 -m venv cellxgene && source cellxgene/bin/activate && yes | pip uninstall cellxgene || true && pip install -e .. && cellxgene launch ",
     "build": "npm run clean && webpack --config configuration/webpack/webpack.config.prod.js",
-    "dev": "npm run clean && webpack --config configuration/webpack/webpack.config.dev.js",
     "clean": "rimraf build",
-    "start": "node server/development.js",
+    "dev": "npm run clean && webpack --config configuration/webpack/webpack.config.dev.js",
+    "e2e": "jest --verbose false --config __tests__/e2e/e2eJestConfig.json e2e",
     "lint": "eslint src",
-    "test": "jest"
+    "smoke-test": "start-server-and-test start-server-for-test :5000 e2e",
+    "start": "node server/development.js",
+    "start-server-for-test": "cellxgene launch -p 5000 ../example-dataset/pbmc3k.h5ad",
+    "test": "jest",
+    "unit-test": "jest --testPathIgnorePatterns e2e"
   },
   "engineStrict": true,
   "engines": {
@@ -66,6 +70,8 @@
     "@babel/plugin-proposal-decorators": "^7.0.0",
     "@babel/plugin-proposal-export-namespace-from": "^7.0.0",
     "@babel/plugin-proposal-function-bind": "^7.0.0",
+    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.2.0",
+    "@babel/plugin-proposal-optional-chaining": "^7.2.0",
     "@babel/plugin-transform-react-constant-elements": "^7.0.0",
     "@babel/plugin-transform-runtime": "^7.1.0",
     "@babel/preset-env": "^7.1.5",
@@ -95,10 +101,13 @@
     "html-webpack-inline-source-plugin": "0.0.10",
     "html-webpack-plugin": "^3.2.0",
     "jest": "^24.1.0",
+    "jest-puppeteer": "^4.1.0",
     "json-loader": "^0.5.4",
     "mini-css-extract-plugin": "^0.4.1",
+    "puppeteer": "^1.12.1",
     "rimraf": "^2.6.3",
     "serve-favicon": "^2.3.0",
+    "start-server-and-test": "^1.7.11",
     "style-loader": "^0.23.1",
     "sw-precache-webpack-plugin": "^0.11.5",
     "url-loader": "^1.1.0",

client/src/actions/index.js

@@ -1,12 +1,11 @@
 // jshint esversion: 6
 import _ from "lodash";
 import * as globals from "../globals";
-import { Universe, kvCache } from "../util/stateManager";
+import { Universe } from "../util/stateManager";
 import {
   catchErrorsWrap,
   doJsonRequest,
   doBinaryRequest,
-  rangeEncodeIndices,
   dispatchNetworkErrorMessageToUser
 } from "../util/actionHelpers";
 
@@ -40,7 +39,7 @@ const doInitialDataLoad = () =>
       /* set config defaults */
       const config = { ...globals.configDefaults, ...results[0].config };
       const [, schema, obsAnno, varAnno, obsLayout] = [...results];
-      const universe = Universe.createUniverseFromRestV02Response(
+      const universe = Universe.createUniverseFromResponse(
         config,
         schema,
         obsAnno,
@@ -66,7 +65,7 @@ Set the view (world) to current selection.   Placeholder for an async action
 which also does re-layout.
 */
 const regraph = () => (dispatch, getState) => {
-  const { universe, world, crossfilter } = getState().controls;
+  const { universe, world, crossfilter } = getState();
   dispatch({
     type: "set World to current selection",
     universe,
@@ -125,14 +124,14 @@ async function _doRequestExpressionData(dispatch, getState, genes) {
   };
 
   const state = getState();
-  const { universe } = state.controls;
+  const { universe } = state;
   /* preload data already in cache */
   let expressionData = _.transform(
     genes,
     (expData, g) => {
-      const data = kvCache.get(universe.varDataCache, g);
+      const data = universe.varData.col(g);
       if (data) {
-        expData[g] = data;
+        expData[g] = data.asArray();
       }
     },
     {}
@@ -165,12 +164,12 @@ function requestSingleGeneExpressionCountsForColoringPOST(gene) {
     dispatch({ type: "get single gene expression for coloring started" });
     try {
       await _doRequestExpressionData(dispatch, getState, [gene]);
-      const { world } = getState().controls;
+      const { world } = getState();
       dispatch({
         type: "color by expression",
         gene,
         data: {
-          [gene]: kvCache.get(world.varDataCache, gene)
+          [gene]: world.varData.col(gene).asArray()
         }
       });
     } catch (error) {
@@ -186,14 +185,14 @@ const requestUserDefinedGene = gene => async (dispatch, getState) => {
   dispatch({ type: "request user defined gene started" });
   try {
     await await _doRequestExpressionData(dispatch, getState, [gene]);
-    const { world } = getState().controls;
+    const { world } = getState();
 
     /* then send the success case action through */
     return dispatch({
       type: "request user defined gene success",
       data: {
         genes: [gene],
-        expression: kvCache.get(world.varDataCache, gene)
+        expression: world.varData.col(gene).asArray()
       }
     });
   } catch (error) {
@@ -241,13 +240,19 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
     2. get expression data for each
     */
     const state = getState();
-    const { universe } = state.controls;
-    const set1ByIndex = rangeEncodeIndices(
-      _.map(set1, s => universe.obsNameToIndexMap[s])
-    );
-    const set2ByIndex = rangeEncodeIndices(
-      _.map(set2, s => universe.obsNameToIndexMap[s])
-    );
+    const { universe } = state;
+
+    // Legal values are null, Array or TypedArray.  Null is initial state.
+    if (!set1) set1 = [];
+    if (!set2) set2 = [];
+
+    // These lines ensure that we convert any TypedArray to an Array.
+    // This is necessary because JSON.stringify() does some very strange
+    // things with TypedArrays (they are marshalled to JSON objects, rather
+    // than being marshalled as a JSON array).
+    set1 = Array.isArray(set1) ? set1 : Array.from(set1);
+    set2 = Array.isArray(set2) ? set2 : Array.from(set2);
+
     const res = await fetch(
       `${globals.API.prefix}${globals.API.version}diffexp/obs`,
       {
@@ -259,8 +264,8 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
         body: JSON.stringify({
           mode: "topN",
           count: num_genes,
-          set1: { filter: { obs: { index: set1ByIndex } } },
-          set2: { filter: { obs: { index: set2ByIndex } } }
+          set1: { filter: { obs: { index: set1 } } },
+          set2: { filter: { obs: { index: set2 } } }
         })
       }
     );
@@ -271,7 +276,9 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
 
     const data = await res.json();
     // result is [ [varIdx, ...], ... ]
-    const topNGenes = _.map(data, r => universe.varAnnotations[r[0]].name);
+    const topNGenes = _.map(data, r =>
+      universe.varAnnotations.at(r[0], "name")
+    );
 
     /*
     Kick off secondary action to fetch all of the expression data for the
@@ -293,7 +300,7 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
 };
 
 const resetInterface = () => (dispatch, getState) => {
-  const { universe } = getState().controls;
+  const { universe } = getState();
 
   dispatch({
     type: "clear all user defined genes"

client/src/components/brushableHistogram/index.js

@@ -10,21 +10,19 @@ import { Button, ButtonGroup, Tooltip } from "@blueprintjs/core";
 import { connect } from "react-redux";
 import * as d3 from "d3";
 import memoize from "memoize-one";
-import { kvCache } from "../../util/stateManager";
 import * as globals from "../../globals";
 import actions from "../../actions";
 import finiteExtent from "../../util/finiteExtent";
+import { makeContinuousDimensionName } from "../../util/nameCreators";
 
 @connect(state => ({
-  world: state.controls.world,
+  world: state.world,
   scatterplotXXaccessor: state.controls.scatterplotXXaccessor,
   scatterplotYYaccessor: state.controls.scatterplotYYaccessor,
-  crossfilter: state.controls.crossfilter,
+  continuousSelection: state.continuousSelection,
   differential: state.differential,
-  initializeRanges: _.get(state.controls.world, "summary.obs"),
-  colorAccessor: state.controls.colorAccessor,
-  colorScale: state.controls.colorScale,
-  obsAnnotations: _.get(state.controls.world, "obsAnnotations", null)
+  colorAccessor: state.colors.colorAccessor,
+  obsAnnotations: _.get(state.world, "obsAnnotations", null)
 }))
 class HistogramBrush extends React.Component {
   calcHistogramCache = memoize((obsAnnotations, field, rangeMin, rangeMax) => {
@@ -35,9 +33,11 @@ class HistogramBrush extends React.Component {
       .scaleLinear()
       .range([this.height - this.marginBottom, 0]);
 
-    if (obsAnnotations[0][field] !== undefined) {
+    if (obsAnnotations.hasCol(field)) {
       // recalculate expensive stuff
-      const allValuesForContinuousFieldAsArray = _.map(obsAnnotations, field);
+      const allValuesForContinuousFieldAsArray = obsAnnotations
+        .col(field)
+        .asArray();
 
       histogramCache.x = d3
         .scaleLinear()
@@ -50,9 +50,8 @@ class HistogramBrush extends React.Component {
         .thresholds(40)(allValuesForContinuousFieldAsArray);
 
       histogramCache.numValues = allValuesForContinuousFieldAsArray.length;
-    } else if (kvCache.get(world.varDataCache, field)) {
-      /* it's not in observations, so it's a gene, but let's check to make sure */
-      const varValues = kvCache.get(world.varDataCache, field);
+    } else if (world.varData.hasCol(field)) {
+      const varValues = world.varData.col(field).asArray();
 
       histogramCache.x = d3
         .scaleLinear()
@@ -88,21 +87,65 @@ class HistogramBrush extends React.Component {
   }
 
   componentDidUpdate(prevProps) {
-    const { field, obsAnnotations } = this.props;
+    const { field, obsAnnotations, continuousSelection } = this.props;
     const { x, y, bins, numValues, svgRef } = this._histogram;
 
     if (obsAnnotations !== prevProps.obsAnnotations) {
       this.renderAxesBrushBins(x, y, bins, numValues, svgRef, field);
     }
+
+    /*
+    if the selection has changed, ensure that the brush correctly reflects
+    the underlying selection.
+    */
+    if (continuousSelection !== prevProps.continuousSelection) {
+      const { isObs, isUserDefined, isDiffExp } = this.props;
+      const myName = makeContinuousDimensionName(
+        { isObs, isUserDefined, isDiffExp },
+        field
+      );
+      const range = continuousSelection[myName];
+      const { brushXselection, brushX } = this.state;
+      if (brushXselection) {
+        const selection = d3.brushSelection(brushXselection.node());
+        if (!range && selection) {
+          /* no active selection - clear brush */
+          brushXselection.call(brushX.move, null);
+        } else if (range && !selection) {
+          /* there is an active selection, but no brush - set the brush */
+          const x0 = x(range[0]);
+          const x1 = x(range[1]);
+          brushXselection.call(brushX.move, [x0, x1]);
+        } else if (range && selection) {
+          /* there is an active selection and a brush - make sure they match */
+          const moveDeltaThreshold = 1;
+          const x0 = x(range[0]);
+          const x1 = x(range[1]);
+          const dX0 = Math.abs(x0 - selection[0]);
+          const dX1 = Math.abs(x1 - selection[1]);
+          /*
+          only update the brush if it is grossly incorrect, 
+          as defined by the moveDeltaThreshold
+          */
+          if (dX0 > moveDeltaThreshold || dX1 > moveDeltaThreshold) {
+            brushXselection.call(brushX.move, [x0, x1]);
+          }
+        }
+      }
+    }
   }
 
-  onBrush(selection, x) {
+  onBrush(selection, x, eventType) {
+    const type = `continuous metadata histogram ${eventType}`;
     return () => {
       const { dispatch, field, isObs, isUserDefined, isDiffExp } = this.props;
 
+      // ignore programmatically generated events
+      if (!d3.event.sourceEvent) return;
+
       if (d3.event.selection) {
         dispatch({
-          type: "continuous metadata histogram brush",
+          type,
           selection: field,
           continuousNamespace: {
             isObs,
@@ -113,7 +156,67 @@ class HistogramBrush extends React.Component {
         });
       } else {
         dispatch({
-          type: "continuous metadata histogram brush",
+          type,
+          selection: field,
+          continuousNamespace: {
+            isObs,
+            isUserDefined,
+            isDiffExp
+          },
+          range: null
+        });
+      }
+    };
+  }
+
+  onBrushEnd(selection, x) {
+    return () => {
+      const { dispatch, field, isObs, isUserDefined, isDiffExp } = this.props;
+      const { brushXselection } = this.state;
+      const minAllowedBrushSize = 10;
+      const smallAmountToAvoidInfiniteLoop = 0.1;
+
+      // ignore programmatically generated events
+      if (!d3.event.sourceEvent) return;
+
+      if (d3.event.selection) {
+        let _range;
+
+        if (
+          d3.event.selection[1] - d3.event.selection[0] >
+          minAllowedBrushSize
+        ) {
+          _range = [x(d3.event.selection[0]), x(d3.event.selection[1])];
+        } else {
+          /* the user selected range is too small and will be hidden #587, so take control of it procedurally */
+          /* https://stackoverflow.com/questions/12354729/d3-js-limit-size-of-brush */
+
+          const procedurallyResizedBrushWidth =
+            d3.event.selection[0] +
+            minAllowedBrushSize +
+            smallAmountToAvoidInfiniteLoop; //
+
+          _range = [x(d3.event.selection[0]), x(procedurallyResizedBrushWidth)];
+
+          d3.event.target.move(brushXselection, [
+            d3.event.selection[0],
+            procedurallyResizedBrushWidth
+          ]);
+        }
+
+        dispatch({
+          type: "continuous metadata histogram end",
+          selection: field,
+          continuousNamespace: {
+            isObs,
+            isUserDefined,
+            isDiffExp
+          },
+          range: _range
+        });
+      } else {
+        dispatch({
+          type: "continuous metadata histogram end",
           selection: field,
           continuousNamespace: {
             isObs,
@@ -141,21 +244,15 @@ class HistogramBrush extends React.Component {
   }
 
   handleColorAction() {
-    const {
-      obsAnnotations,
-      dispatch,
-      field,
-      world,
-      initializeRanges
-    } = this.props;
+    const { obsAnnotations, dispatch, field, world, ranges } = this.props;
 
-    if (obsAnnotations[0][field]) {
+    if (obsAnnotations.hasCol(field)) {
       dispatch({
         type: "color by continuous metadata",
         colorAccessor: field,
-        rangeMaxForColorAccessor: initializeRanges[field].range.max
+        rangeForColorAccessor: ranges
       });
-    } else if (kvCache.get(world.varDataCache, field)) {
+    } else if (world.varData.hasCol(field)) {
       dispatch(actions.requestSingleGeneExpressionCountsForColoringPOST(field));
     }
   }
@@ -232,15 +329,21 @@ class HistogramBrush extends React.Component {
       .attr("height", d => y(0) - y(d.length / numValues));
 
     /* BRUSH */
-    d3.select(svgRef)
+    const brushX = d3
+      .brushX()
+      /*
+      emit start so that the Undoable history can save an undo point
+      upon drag start, and ignore the subsequent intermediate drag events.
+      */
+      .on("start", this.onBrush(field, x.invert, "start").bind(this))
+      .on("brush", this.onBrush(field, x.invert, "brush").bind(this))
+      .on("end", this.onBrushEnd(field, x.invert).bind(this));
+    const brushXselection = d3
+      .select(svgRef)
       .append("g")
       .attr("class", "brush")
-      .call(
-        d3
-          .brushX()
-          .on("brush", this.onBrush(field, x.invert).bind(this))
-          .on("end", this.onBrush(field, x.invert).bind(this))
-      );
+      .attr("data-testid", `${svgRef.dataset.testid}-brush`)
+      .call(brushX);
 
     /* AXIS */
     d3.select(svgRef)
@@ -260,6 +363,8 @@ class HistogramBrush extends React.Component {
     d3.select(svgRef)
       .selectAll(".axis--x line")
       .style("stroke", "rgb(230,230,230)");
+
+    this.setState({ brushX, brushXselection });
   }
 
   render() {
@@ -275,10 +380,18 @@ class HistogramBrush extends React.Component {
       scatterplotYYaccessor,
       zebra
     } = this.props;
-
+    const field_for_id = field.replace(/\s/g, "_");
     return (
       <div
-        id={`histogram_${field}`}
+        id={`histogram_${field_for_id}`}
+        data-testid={`histogram-${field}`}
+        data-testclass={
+          isDiffExp
+            ? "histogram-diffexp"
+            : isUserDefined
+            ? "histogram-user-gene"
+            : "histogram-continuous-metadata"
+        }
         style={{
           padding: globals.leftSidebarSectionPadding,
           backgroundColor: zebra ? globals.lightestGrey : "white"
@@ -293,6 +406,7 @@ class HistogramBrush extends React.Component {
               />
               <ButtonGroup style={{ marginRight: 7 }}>
                 <Button
+                  data-testid={`plot-x-${field}`}
                   onClick={this.handleSetGeneAsScatterplotX(field).bind(this)}
                   active={scatterplotXXaccessor === field}
                   intent={scatterplotXXaccessor === field ? "primary" : "none"}
@@ -300,6 +414,7 @@ class HistogramBrush extends React.Component {
                   plot x
                 </Button>
                 <Button
+                  data-testid={`plot-y-${field}`}
                   onClick={this.handleSetGeneAsScatterplotY(field).bind(this)}
                   active={scatterplotYYaccessor === field}
                   intent={scatterplotYYaccessor === field ? "primary" : "none"}
@@ -327,6 +442,8 @@ class HistogramBrush extends React.Component {
               onClick={this.handleColorAction.bind(this)}
               active={colorAccessor === field}
               intent={colorAccessor === field ? "primary" : "none"}
+              data-testclass="colorby"
+              data-testid={`colorby-${field}`}
               icon="tint"
             />
           </Tooltip>
@@ -334,7 +451,9 @@ class HistogramBrush extends React.Component {
         <svg
           width={this.width}
           height={this.height}
-          id={`histogram_${field}_svg`}
+          id={`histogram_${field_for_id}_svg`}
+          data-testclass="histogram-plot"
+          data-testid={`histogram-${field}-plot`}
           ref={svgRef => {
             this.drawHistogram(svgRef);
           }}
@@ -345,7 +464,12 @@ class HistogramBrush extends React.Component {
             justifyContent: "center"
           }}
         >
-          <span style={{ fontStyle: "italic" }}>{field}</span>
+          <span
+            data-testclass="brushable-histogram-field-name"
+            style={{ fontStyle: "italic" }}
+          >
+            {field}
+          </span>
         </div>
 
         {isDiffExp ? (

client/src/components/categorical/categorical.js

@@ -6,12 +6,12 @@ import * as globals from "../../globals";
 import Category from "./category";
 
 @connect(state => ({
-  categoricalSelectionState: state.controls.categoricalSelectionState
+  categoricalSelection: state.categoricalSelection
 }))
 class Categories extends React.Component {
   render() {
-    const { categoricalSelectionState } = this.props;
-    if (!categoricalSelectionState) return null;
+    const { categoricalSelection } = this.props;
+    if (!categoricalSelection) return null;
 
     return (
       <div
@@ -26,7 +26,7 @@ class Categories extends React.Component {
         >
           Categorical Metadata
         </p>
-        {_.map(categoricalSelectionState, (catState, catName) => (
+        {_.map(categoricalSelection, (catState, catName) => (
           <Category key={catName} metadataField={catName} />
         ))}
       </div>

client/src/components/categorical/category.js

@@ -9,8 +9,8 @@ import Value from "./value";
 import sortedCategoryValues from "./util";
 
 @connect(state => ({
-  colorAccessor: state.controls.colorAccessor,
-  categoricalSelectionState: state.controls.categoricalSelectionState
+  colorAccessor: state.colors.colorAccessor,
+  categoricalSelection: state.categoricalSelection
 }))
 class Category extends React.Component {
   constructor(props) {
@@ -21,38 +21,43 @@ class Category extends React.Component {
     };
   }
 
-  componentDidUpdate() {
-    const { categoricalSelectionState, metadataField } = this.props;
-    const cat = categoricalSelectionState[metadataField];
-    const categoryCount = {
-      // total number of categories in this dimension
-      totalCatCount: cat.numCategories,
-      // number of selected options in this category
-      selectedCatCount: _.reduce(
-        cat.categorySelected,
-        (res, cond) => (cond ? res + 1 : res),
-        0
-      )
-    };
-    if (categoryCount.selectedCatCount === categoryCount.totalCatCount) {
-      /* everything is on, so not indeterminate */
-      this.checkbox.indeterminate = false;
-    } else if (categoryCount.selectedCatCount === 0) {
-      /* nothing is on, so no */
-      this.checkbox.indeterminate = false;
-    } else if (categoryCount.selectedCatCount < categoryCount.totalCatCount) {
-      /* to be explicit... */
-      this.checkbox.indeterminate = true;
+  componentDidUpdate(prevProps) {
+    const { categoricalSelection, metadataField } = this.props;
+    if (categoricalSelection !== prevProps.categoricalSelection) {
+      const cat = categoricalSelection[metadataField];
+      const categoryCount = {
+        // total number of categories in this dimension
+        totalCatCount: cat.numCategories,
+        // number of selected options in this category
+        selectedCatCount: _.reduce(
+          cat.categorySelected,
+          (res, cond) => (cond ? res + 1 : res),
+          0
+        )
+      };
+      if (categoryCount.selectedCatCount === categoryCount.totalCatCount) {
+        /* everything is on, so not indeterminate */
+        this.checkbox.indeterminate = false;
+        this.setState({ isChecked: true }); // eslint-disable-line react/no-did-update-set-state
+      } else if (categoryCount.selectedCatCount === 0) {
+        /* nothing is on, so no */
+        this.checkbox.indeterminate = false;
+        this.setState({ isChecked: false }); // eslint-disable-line react/no-did-update-set-state
+      } else if (categoryCount.selectedCatCount < categoryCount.totalCatCount) {
+        /* to be explicit... */
+        this.checkbox.indeterminate = true;
+        this.setState({ isChecked: false });
+      }
     }
   }
 
-  handleColorChange() {
+  handleColorChange = () => {
     const { dispatch, metadataField } = this.props;
     dispatch({
       type: "color by categorical metadata",
       colorAccessor: metadataField
     });
-  }
+  };
 
   toggleAll() {
     const { dispatch, metadataField } = this.props;
@@ -83,9 +88,9 @@ class Category extends React.Component {
   }
 
   renderCategoryItems() {
-    const { categoricalSelectionState, metadataField } = this.props;
+    const { categoricalSelection, metadataField } = this.props;
 
-    const cat = categoricalSelectionState[metadataField];
+    const cat = categoricalSelection[metadataField];
     const optTuples = sortedCategoryValues([...cat.categoryIndices]);
     return _.map(optTuples, (tuple, i) => (
       <Value
@@ -100,17 +105,15 @@ class Category extends React.Component {
 
   render() {
     const { isExpanded, isChecked } = this.state;
-    const {
-      metadataField,
-      colorAccessor,
-      categoricalSelectionState
-    } = this.props;
-    const { isTruncated } = categoricalSelectionState[metadataField];
+    const { metadataField, colorAccessor, categoricalSelection } = this.props;
+    const { isTruncated } = categoricalSelection[metadataField];
     return (
       <div
         style={{
           maxWidth: globals.maxControlsWidth
         }}
+        data-testclass="category"
+        data-testid={`category-${metadataField}`}
       >
         <div
           style={{
@@ -128,6 +131,8 @@ class Category extends React.Component {
           >
             <label className="bp3-control bp3-checkbox">
               <input
+                data-testclass="category-select"
+                data-testid={`category-select-${metadataField}`}
                 onChange={this.handleToggleAllClick.bind(this)}
                 ref={el => {
                   this.checkbox = el;
@@ -141,6 +146,7 @@ class Category extends React.Component {
             </label>
 
             <span
+              data-testid={`category-expand-${metadataField}`}
               style={{
                 cursor: "pointer",
                 display: "inline-block"
@@ -151,18 +157,26 @@ class Category extends React.Component {
             >
               {metadataField}
               {isExpanded ? (
-                <FaChevronDown style={{ fontSize: 10, marginLeft: 5 }} />
+                <FaChevronDown
+                  data-testclass="category-expand-is-expanded"
+                  style={{ fontSize: 10, marginLeft: 5 }}
+                />
               ) : (
-                <FaChevronRight style={{ fontSize: 10, marginLeft: 5 }} />
+                <FaChevronRight
+                  data-testclass="category-expand-is-not-expanded"
+                  style={{ fontSize: 10, marginLeft: 5 }}
+                />
               )}
             </span>
           </div>
           <Tooltip content="Use as color scale" position="bottom">
             <Button
-              onClick={this.handleColorChange.bind(this)}
+              data-testclass="colorby"
+              data-testid={`colorby-${metadataField}`}
+              onClick={this.handleColorChange}
               active={colorAccessor === metadataField}
               intent={colorAccessor === metadataField ? "primary" : "none"}
-              icon={"tint"}
+              icon="tint"
             />
           </Tooltip>
         </div>

client/src/components/categorical/occupancy.js

@@ -10,7 +10,7 @@ class Occupancy extends React.Component {
     const {
       occupancy,
       colorScale,
-      categoricalSelectionState,
+      categoricalSelection,
       colorAccessor,
       schema
     } = this.props;
@@ -29,23 +29,21 @@ class Occupancy extends React.Component {
 
     let currentOffset = 0;
 
-    const stacks = categoricalSelectionState[colorAccessor].categoryValues.map(
-      d => {
-        const o = occupancy.get(d);
+    const stacks = categoricalSelection[colorAccessor].categoryValues.map(d => {
+      const o = occupancy.get(d);
 
-        const scaledValue = x(o);
+      const scaledValue = x(o);
 
-        const stackItem = {
-          key: d,
-          value: o || 0,
-          rectWidth: o ? scaledValue : 0,
-          offset: currentOffset,
-          fill: o ? colorScale(categories.indexOf(d)) : "rgb(255,255,255)"
-        };
-        currentOffset += o ? scaledValue : 0;
-        return stackItem;
-      }
-    );
+      const stackItem = {
+        key: d,
+        value: o || 0,
+        rectWidth: o ? scaledValue : 0,
+        offset: currentOffset,
+        fill: o ? colorScale(categories.indexOf(d)) : "rgb(255,255,255)"
+      };
+      currentOffset += o ? scaledValue : 0;
+      return stackItem;
+    });
 
     return (
       <svg

client/src/components/categorical/value.js

@@ -7,11 +7,11 @@ import { countCategoryValues2D } from "../../util/stateManager/worldUtil";
 import * as globals from "../../globals";
 
 @connect(state => ({
-  categoricalSelectionState: state.controls.categoricalSelectionState,
-  colorScale: state.controls.colorScale,
-  colorAccessor: state.controls.colorAccessor,
-  schema: _.get(state.controls.world, "schema", null),
-  world: state.controls.world
+  categoricalSelection: state.categoricalSelection,
+  colorScale: state.colors.scale,
+  colorAccessor: state.colors.colorAccessor,
+  schema: _.get(state.world, "schema", null),
+  world: state.world
 }))
 class CategoryValue extends React.Component {
   toggleOff() {
@@ -34,7 +34,7 @@ class CategoryValue extends React.Component {
 
   render() {
     const {
-      categoricalSelectionState,
+      categoricalSelection,
       metadataField,
       categoryIndex,
       colorAccessor,
@@ -44,9 +44,9 @@ class CategoryValue extends React.Component {
       world
     } = this.props;
 
-    if (!categoricalSelectionState) return null;
+    if (!categoricalSelection) return null;
 
-    const category = categoricalSelectionState[metadataField];
+    const category = categoricalSelection[metadataField];
     const selected = category.categorySelected[categoryIndex];
     const count = category.categoryCounts[categoryIndex];
     const value = category.categoryValues[categoryIndex];
@@ -65,7 +65,7 @@ class CategoryValue extends React.Component {
       })[0].categories;
     }
 
-    if (colorAccessor && !isColorBy) {
+    if (colorAccessor && !isColorBy && categoricalSelection[colorAccessor]) {
       occupancy = countCategoryValues2D(
         metadataField,
         colorAccessor,
@@ -81,6 +81,7 @@ class CategoryValue extends React.Component {
           alignItems: "baseline",
           justifyContent: "space-between"
         }}
+        data-testclass="categorical-row"
       >
         <div
           style={{
@@ -97,16 +98,23 @@ class CategoryValue extends React.Component {
               onChange={
                 selected ? this.toggleOff.bind(this) : this.toggleOn.bind(this)
               }
+              data-testclass="categorical-value-select"
+              data-testid={`categorical-value-select-${metadataField}-${displayString}`}
               checked={selected}
               type="checkbox"
             />
             <span className="bp3-control-indicator" />
-            {displayString}
+            <span
+              data-testid={`categorical-value-${metadataField}-${displayString}`}
+              data-testclass="categorical-value"
+            >
+              {displayString}
+            </span>
           </label>
           <span style={{ flexShrink: 0 }}>
             {colorAccessor &&
             !isColorBy &&
-            categoricalSelectionState[colorAccessor] ? (
+            categoricalSelection[colorAccessor] ? (
               <Occupancy
                 occupancy={occupancy.get(
                   category.categoryValues[categoryIndex]
@@ -117,7 +125,12 @@ class CategoryValue extends React.Component {
           </span>
         </div>
         <span>
-          <span>{count}</span>
+          <span
+            data-testclass="categorical-value-count"
+            data-testid={`categorical-value-count-${metadataField}-${displayString}`}
+          >
+            {count}
+          </span>
           <svg
             style={{
               marginLeft: 5,

client/src/components/continuous/continuous.js

@@ -9,12 +9,10 @@ import * as globals from "../../globals";
 import HistogramBrush from "../brushableHistogram";
 
 @connect(state => ({
-  ranges: _.get(state.controls.world, "summary.obs", null),
-  metadata: _.get(state.controls.world, "obsAnnotations", null),
-  colorAccessor: state.controls.colorAccessor,
-  colorScale: state.controls.colorScale,
-  selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null),
-  schema: _.get(state.controls.world, "schema", null)
+  obsAnnotations: _.get(state.world, "obsAnnotations", null),
+  colorAccessor: state.colors.colorAccessor,
+  colorScale: state.colors.scale,
+  schema: _.get(state.world, "schema", null)
 }))
 class Continuous extends React.Component {
   constructor(props) {
@@ -29,17 +27,18 @@ class Continuous extends React.Component {
 
   handleColorAction(key) {
     return () => {
-      const { dispatch, ranges } = this.props;
+      const { dispatch, obsAnnotations } = this.props;
+      const summary = obsAnnotations.col(key).summarize();
       dispatch({
         type: "color by continuous metadata",
         colorAccessor: key,
-        rangeMaxForColorAccessor: ranges[key].range.max
+        rangeForColorAccessor: summary
       });
     };
   }
 
   render() {
-    const { ranges, obsAnnotations, schema } = this.props;
+    const { obsAnnotations, schema } = this.props;
     if (schema && !this.continuousChecked) {
       this.hasContinuous = _.some(
         schema.annotations.obs,
@@ -63,24 +62,34 @@ class Continuous extends React.Component {
             Continuous metadata
           </p>
         ) : null}
-        {_.map(ranges, (value, key) => {
-          const isColorField = key.includes("color") || key.includes("Color");
-          zebra += 1;
-          if (value.range && key !== "name" && !isColorField) {
-            return (
-              <HistogramBrush
-                key={key}
-                field={key}
-                isObs
-                zebra={zebra % 2 === 0}
-                fieldValues={obsAnnotations}
-                ranges={value.range}
-                handleColorAction={this.handleColorAction(key).bind(this)}
-              />
-            );
-          }
-          return null;
-        })}
+        {obsAnnotations
+          ? _.map(obsAnnotations.colIndex.keys(), key => {
+              const summary = obsAnnotations.col(key).summarize();
+              const isColorField =
+                key.includes("color") || key.includes("Color");
+              const nonFiniteExtent =
+                summary.min === undefined || summary.max === undefined;
+              zebra += 1;
+              if (
+                !summary.categorical &&
+                key !== "name" &&
+                !isColorField &&
+                !nonFiniteExtent
+              ) {
+                return (
+                  <HistogramBrush
+                    key={key}
+                    field={key}
+                    isObs
+                    zebra={zebra % 2 === 0}
+                    ranges={summary}
+                    handleColorAction={this.handleColorAction(key).bind(this)}
+                  />
+                );
+              }
+              return null;
+            })
+          : null}
       </div>
     );
   }

client/src/components/continuousLegend/index.js

@@ -98,8 +98,8 @@ const continuous = (selectorId, colorscale, colorAccessor) => {
 };
 
 @connect(state => ({
-  colorAccessor: state.controls.colorAccessor,
-  colorScale: state.controls.colorScale,
+  colorAccessor: state.colors.colorAccessor,
+  colorScale: state.colors.scale,
   responsive: state.responsive
 }))
 class ContinuousLegend extends React.Component {

client/src/components/geneExpression/cellSetButtons.js

@@ -1,8 +1,8 @@
 // jshint esversion: 6
 import React from "react";
-import _ from "lodash";
 import { AnchorButton, Tooltip } from "@blueprintjs/core";
 import { connect } from "react-redux";
+import { World } from "../../util/stateManager";
 
 @connect()
 class CellSetButton extends React.Component {
@@ -14,7 +14,11 @@ class CellSetButton extends React.Component {
       eitherCellSetOneOrTwo
     } = this.props;
 
-    const set = _.map(crossfilter.allFiltered(), "name");
+    // Reducer and components assume that value will be null if
+    // no selection made.  World..getSelectedByIndex() returns a
+    // zero length TypedArray when nothing is selected.
+    let set = World.getSelectedByIndex(crossfilter);
+    if (set.length === 0) set = null;
 
     if (!differential.diffExp) {
       /* diffexp needs to be cleared before we store a new set */
@@ -28,6 +32,9 @@ class CellSetButton extends React.Component {
   render() {
     const { differential, eitherCellSetOneOrTwo } = this.props;
     const cellListName = `celllist${eitherCellSetOneOrTwo}`;
+    let cells_selected = differential[cellListName]
+      ? differential[cellListName].length
+      : 0;
     return (
       <Tooltip
         content="Save current selection for differential expression computation"
@@ -38,12 +45,14 @@ class CellSetButton extends React.Component {
           type="button"
           disabled={differential.diffExp}
           onClick={this.set.bind(this)}
+          data-testid={`cellset-button-${eitherCellSetOneOrTwo}`}
         >
           {eitherCellSetOneOrTwo}
           {": "}
-          {differential[cellListName]
-            ? `${differential[cellListName].length} cells`
-            : "0 cells"}
+          <span data-testid={`cellset-count-${eitherCellSetOneOrTwo}`}>
+            {cells_selected}
+          </span>
+          {" cells"}
         </AnchorButton>
       </Tooltip>
     );

client/src/components/geneExpression/expressionButtons.js

@@ -9,9 +9,8 @@ import CellSetButton from "./cellSetButtons";
 
 @connect(state => ({
   differential: state.differential,
-  world: state.controls.world,
-  crossfilter: state.controls.crossfilter,
-  selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null)
+  world: state.world,
+  crossfilter: state.crossfilter
 }))
 class Expression extends React.Component {
   constructor(props) {
@@ -68,6 +67,7 @@ class Expression extends React.Component {
               style={{ marginTop: 10 }}
               disabled={!haveBothCellSets}
               intent="primary"
+              data-testid="diffexp-button"
               loading={differential.loading}
               fill
               type="button"

client/src/components/geneExpression/index.js

@@ -29,49 +29,40 @@ const renderGene = (fuzzySortResult, { handleClick, modifiers, query }) => {
     return null;
   }
   /* the fuzzysort wraps the object with other properties, like a score */
-  const gene = fuzzySortResult.obj;
-  const text = gene.name;
+  const geneName = fuzzySortResult.target;
 
   return (
     <MenuItem
       active={modifiers.active}
       disabled={modifiers.disabled}
+      data-testid={`suggest-menu-item-${geneName}`}
       // Use of annotations in this way is incorrect and dataset specific.
       // See https://github.com/chanzuckerberg/cellxgene/issues/483
       // label={gene.n_counts}
-      key={gene.name}
-      onClick={g => {
+      key={geneName}
+      onClick={g =>
         /* this fires when user clicks a menu item */
-        handleClick(g);
-      }}
-      text={text}
+        handleClick(g)
+      }
+      text={geneName}
     />
   );
 };
 
-const filterGenes = (query, genes) => {
+const filterGenes = (query, genes) =>
   /* fires on load, once, and then for each character typed into the input */
-  return fuzzysort.go(query, genes, {
-    key: "name",
+  fuzzysort.go(query, genes, {
     limit: 5,
     threshold: -10000 // don't return bad results
   });
-};
 
 @connect(state => {
-  const metadata = _.get(state.controls.world, "obsAnnotations", null);
-  const ranges = _.get(state.controls.world, "summary.obs", null);
-  const initializeRanges = _.get(state.controls.world, "summary.obs");
-
   return {
-    ranges,
-    metadata,
-    initializeRanges,
+    obsAnnotations: _.get(state.world, "obsAnnotations", null),
     userDefinedGenes: state.controls.userDefinedGenes,
     userDefinedGenesLoading: state.controls.userDefinedGenesLoading,
-    world: state.controls.world,
-    colorAccessor: state.controls.colorAccessor,
-    allGeneNames: state.controls.allGeneNames,
+    world: state.world,
+    colorAccessor: state.colors.colorAccessor,
     differential: state.differential
   };
 })
@@ -84,6 +75,37 @@ class GeneExpression extends React.Component {
     };
   }
 
+  placeholderGeneNames() {
+    /*
+    return a string containing gene name suggestions for use as a user hint.
+    Eg.,    Apod, Cd74, ...
+    Will return a max of 3 genes, totalling 15 characters in length.
+    Randomly selects gene names.
+
+    NOTE: the random selection means it will re-render constantly.
+    */
+    const { world } = this.props;
+    const { varAnnotations } = world;
+    const geneNames = varAnnotations.col("name").asArray();
+    if (geneNames.length > 0) {
+      const placeholder = [];
+      let len = geneNames.length;
+      const maxGeneNameCount = 3;
+      const maxStrLength = 15;
+      len = len < maxGeneNameCount ? len : maxGeneNameCount;
+      for (let i = 0, strLen = 0; i < len && strLen < maxStrLength; i += 1) {
+        const deal = Math.floor(Math.random() * geneNames.length);
+        const geneName = geneNames[deal];
+        placeholder.push(geneName);
+        strLen += geneName.length + 2; // '2' is the length of a comma and space
+      }
+      placeholder.push("...");
+      return placeholder.join(", ");
+    }
+    // default - should never happen.
+    return "Apod, Cd74, ...";
+  }
+
   handleClick(g) {
     const { world, dispatch, userDefinedGenes } = this.props;
     const gene = g.target;
@@ -93,14 +115,12 @@ class GeneExpression extends React.Component {
       postUserErrorToast(
         "That's too many genes, you can have at most 15 user defined genes"
       );
-    } else if (!_.find(world.varAnnotations, { name: gene })) {
+    } else if (world.varAnnotations.col("name").indexOf(gene) === undefined) {
       postUserErrorToast("That doesn't appear to be a valid gene name.");
     } else {
+      dispatch({ type: "single user defined gene start" });
       dispatch(actions.requestUserDefinedGene(gene));
-      dispatch({
-        type: "user defined gene",
-        data: gene
-      });
+      dispatch({ type: "single user defined gene complete" });
     }
   }
 
@@ -115,21 +135,23 @@ class GeneExpression extends React.Component {
     if (bulkAdd !== "") {
       const genes = _.pull(_.uniq(bulkAdd.split(/[ ,]+/)), "");
 
+      dispatch({ type: "bulk user defined gene start" });
       genes.forEach(gene => {
-        if (userDefinedGenes.indexOf(gene) !== -1) {
+        if (gene.length === 0) {
+          keepAroundErrorToast("Must enter a gene name.");
+        } else if (userDefinedGenes.indexOf(gene) !== -1) {
           keepAroundErrorToast("That gene already exists");
-        } else if (!_.find(world.varAnnotations, { name: gene })) {
+        } else if (
+          world.varAnnotations.col("name").indexOf(gene) === undefined
+        ) {
           keepAroundErrorToast(
             `${gene} doesn't appear to be a valid gene name.`
           );
         } else {
           dispatch(actions.requestUserDefinedGene(gene));
-          dispatch({
-            type: "user defined gene",
-            data: gene
-          });
         }
       });
+      dispatch({ type: "bulk user defined gene complete" });
     }
 
     this.setState({ bulkAdd: "" });
@@ -170,6 +192,7 @@ class GeneExpression extends React.Component {
               style={{ marginRight: 5 }}
               minimal
               small
+              data-testid="tab-autosuggest"
               onClick={() => {
                 this.setState({ tab: "autosuggest" });
               }}
@@ -180,6 +203,7 @@ class GeneExpression extends React.Component {
               active={tab === "bulkadd"}
               minimal
               small
+              data-testid="section-bulk-add"
               onClick={() => {
                 this.setState({ tab: "bulkadd" });
               }}
@@ -207,6 +231,7 @@ class GeneExpression extends React.Component {
                   /* this happens on 'enter' */
                   this.handleClick(g);
                 }}
+                inputProps={{ "data-testid": "gene-search" }}
                 inputValueRenderer={g => {
                   return "";
                 }}
@@ -214,13 +239,14 @@ class GeneExpression extends React.Component {
                 itemRenderer={renderGene.bind(this)}
                 items={
                   world && world.varAnnotations
-                    ? world.varAnnotations
-                    : [{ name: "No genes" }]
+                    ? world.varAnnotations.col("name").asArray()
+                    : ["No genes"]
                 }
                 popoverProps={{ minimal: true }}
               />
               <Button
                 className="bp3-button bp3-intent-primary"
+                data-testid={"add-gene"}
                 loading={userDefinedGenesLoading}
               >
                 Add
@@ -245,7 +271,8 @@ class GeneExpression extends React.Component {
                         this.setState({ bulkAdd: e.target.value });
                       }}
                       id="text-input-bulk-add"
-                      placeholder="Apod, Cd74, ..."
+                      data-testid="input-bulk-add"
+                      placeholder={this.placeholderGeneNames()}
                       value={bulkAdd}
                     />
                     <Button
@@ -262,16 +289,17 @@ class GeneExpression extends React.Component {
           ) : null}
           {world && userDefinedGenes.length > 0
             ? _.map(userDefinedGenes, (geneName, index) => {
-                const values = world.varDataCache[geneName];
+                const values = world.varData.col(geneName);
                 if (!values) {
                   return null;
                 }
+                const summary = values.summarize();
                 return (
                   <HistogramBrush
                     key={geneName}
                     field={geneName}
                     zebra={index % 2 === 0}
-                    ranges={finiteExtent(values)}
+                    ranges={summary}
                     isUserDefined
                   />
                 );
@@ -290,18 +318,18 @@ class GeneExpression extends React.Component {
           <ExpressionButtons />
           {differential.diffExp
             ? _.map(differential.diffExp, (value, index) => {
-                const annotations = world.varAnnotations[value[0]];
-                const { name } = annotations;
-                const values = world.varDataCache[name];
+                const name = world.varAnnotations.at(value[0], "name");
+                const values = world.varData.col(name);
                 if (!values) {
                   return null;
                 }
+                const summary = values.summarize();
                 return (
                   <HistogramBrush
                     key={name}
                     field={name}
                     zebra={index % 2 === 0}
-                    ranges={finiteExtent(values)}
+                    ranges={summary}
                     isDiffExp
                     logFoldChange={value[1]}
                     pval={value[2]}

client/src/components/graph/graph.js

@@ -5,25 +5,44 @@ import * as d3 from "d3";
 import { connect } from "react-redux";
 import mat4 from "gl-mat4";
 import _regl from "regl";
-import { Button, AnchorButton, Tooltip } from "@blueprintjs/core";
+import {
+  Button,
+  AnchorButton,
+  Tooltip,
+  Popover,
+  Menu,
+  MenuItem,
+  Position
+} from "@blueprintjs/core";
+
 import * as globals from "../../globals";
 import setupSVGandBrushElements from "./setupSVGandBrush";
 import actions from "../../actions";
 import _camera from "../../util/camera";
 import _drawPoints from "./drawPointsRegl";
 import scaleLinear from "../../util/scaleLinear";
+import { World } from "../../util/stateManager";
 
 /* https://bl.ocks.org/mbostock/9078690 - quadtree for onClick / hover selections */
 
 @connect(state => ({
-  world: state.controls.world,
-  universe: state.controls.universe,
-  crossfilter: state.controls.crossfilter,
+  world: state.world,
+  universe: state.universe,
+  crossfilter: state.crossfilter,
   responsive: state.responsive,
-  colorRGB: _.get(state.controls, "colorRGB", null),
+  colorRGB: state.colors.rgb,
   opacityForDeselectedCells: state.controls.opacityForDeselectedCells,
-  selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null),
-  resettingInterface: state.controls.resettingInterface
+  resettingInterface: state.controls.resettingInterface,
+  userDefinedGenes: state.controls.userDefinedGenes,
+  diffexpGenes: state.controls.diffexpGenes,
+  colorAccessor: state.colors.colorAccessor,
+  scatterplotXXaccessor: state.controls.scatterplotXXaccessor,
+  scatterplotYYaccessor: state.controls.scatterplotYYaccessor,
+  celllist1: state.differential.celllist1,
+  celllist2: state.differential.celllist2,
+  library_versions: _.get(state.config, "library_versions", null),
+  undoDisabled: state["@@undoable/past"].length === 0,
+  redoDisabled: state["@@undoable/future"].length === 0
 }))
 class Graph extends React.Component {
   constructor(props) {
@@ -35,7 +54,8 @@ class Graph extends React.Component {
     this.graphPaddingRight = globals.leftSidebarWidth;
     this.renderCache = {
       positions: null,
-      colors: null
+      colors: null,
+      sizes: null
     };
     this.state = {
       svg: null,
@@ -83,13 +103,8 @@ class Graph extends React.Component {
   }
 
   componentDidUpdate(prevProps) {
-    const {
-      world,
-      crossfilter,
-      selectionUpdate,
-      colorRGB,
-      responsive
-    } = this.props;
+    const { renderCache } = this;
+    const { world, crossfilter, colorRGB, responsive } = this.props;
     const {
       reglRender,
       mode,
@@ -109,35 +124,27 @@ class Graph extends React.Component {
 
     if (regl && world) {
       /* update the regl state */
-      const { obsLayout } = world;
-      const cellCount = crossfilter.size();
+      const { obsLayout, nObs } = world;
+      const X = obsLayout.col("X").asArray();
+      const Y = obsLayout.col("Y").asArray();
 
       // X/Y positions for each point - a cached value that only
       // changes if we have loaded entirely new cell data
       //
-      if (
-        !this.renderCache.positions ||
-        selectionUpdate !== prevProps.selectionUpdate
-      ) {
-        if (!this.renderCache.positions) {
-          this.renderCache.positions = new Float32Array(2 * cellCount);
-        }
+      if (!renderCache.positions || world !== prevProps.world) {
+        renderCache.positions = new Float32Array(2 * nObs);
 
         const glScaleX = scaleLinear([0, 1], [-1, 1]);
         const glScaleY = scaleLinear([0, 1], [1, -1]);
 
-        const offset = [d3.mean(obsLayout.X) - 0.5, d3.mean(obsLayout.Y) - 0.5];
+        const offset = [d3.mean(X) - 0.5, d3.mean(Y) - 0.5];
 
-        for (
-          let i = 0, { positions } = this.renderCache;
-          i < cellCount;
-          i += 1
-        ) {
-          positions[2 * i] = glScaleX(obsLayout.X[i] - offset[0]);
-          positions[2 * i + 1] = glScaleY(obsLayout.Y[i] - offset[1]);
+        for (let i = 0, { positions } = renderCache; i < nObs; i += 1) {
+          positions[2 * i] = glScaleX(X[i] - offset[0]);
+          positions[2 * i + 1] = glScaleY(Y[i] - offset[1]);
         }
         pointBuffer({
-          data: this.renderCache.positions,
+          data: renderCache.positions,
           dimension: 2
         });
 
@@ -147,35 +154,29 @@ class Graph extends React.Component {
       }
 
       // Colors for each point - a cached value that only changes when
-      // the cell metadata changes (done by updateCellColors middleware).
-      // NOTE: this is a slightly pessimistic assumption, as the metadata
-      // could have changed for some other reason, but for now color is
-      // the only metadata that changes client-side.  If this is problematic,
-      // we could add some sort of color-specific indicator to the app state.
-      if (!this.renderCache.colors || colorRGB !== prevProps.colorRGB) {
+      // the cell metadata changes.
+      if (!renderCache.colors || colorRGB !== prevProps.colorRGB) {
         const rgb = colorRGB;
-        if (!this.renderCache.colors) {
-          this.renderCache.colors = new Float32Array(3 * rgb.length);
+        if (!renderCache.colors) {
+          renderCache.colors = new Float32Array(3 * rgb.length);
         }
-        for (let i = 0, { colors } = this.renderCache; i < rgb.length; i += 1) {
+        for (let i = 0, { colors } = renderCache; i < rgb.length; i += 1) {
           colors.set(rgb[i], 3 * i);
         }
-        colorBuffer({ data: this.renderCache.colors, dimension: 3 });
+        colorBuffer({ data: renderCache.colors, dimension: 3 });
       }
 
-      // Sizes for each point - this is presumed to change each time the
-      // component receives new props.   Almost always a true assumption, as
-      // most property upates are due to changes driving a crossfilter
-      // selection set change.
-      //
-      if (!this.renderCache.sizes) {
-        this.renderCache.sizes = new Float32Array(cellCount);
+      // Sizes for each point - updates are triggered only when selected
+      // obs change
+      if (!renderCache.sizes || crossfilter !== prevProps.crossfilter) {
+        if (!renderCache.sizes) {
+          renderCache.sizes = new Float32Array(nObs);
+        }
+        crossfilter.fillByIsSelected(renderCache.sizes, 4, 0.2);
+        sizeBuffer({ data: renderCache.sizes, dimension: 1 });
       }
 
-      crossfilter.fillByIsFiltered(this.renderCache.sizes, 4, 0.2);
-      sizeBuffer({ data: this.renderCache.sizes, dimension: 1 });
-
-      this.count = cellCount;
+      this.count = nObs;
 
       regl._refresh();
       this.reglDraw(
@@ -210,6 +211,56 @@ class Graph extends React.Component {
     }
   }
 
+  isResetDisabled = () => {
+    /*
+    Reset should be disabled when all of the following are true:
+      * nothing is selected in the crossfilter
+      * world EQ universe
+      * nothing is colored by
+      * there are no userDefinedGenes or diffexpGenes displayed
+      * scatterplot is not displayed
+      * nothing in cellset1 or cellset2
+    */
+    const {
+      crossfilter,
+      world,
+      universe,
+      userDefinedGenes,
+      diffexpGenes,
+      colorAccessor,
+      scatterplotXXaccessor,
+      scatterplotYYaccessor,
+      celllist1,
+      celllist2
+    } = this.props;
+
+    if (!crossfilter || !world || !universe) {
+      return false;
+    }
+    const nothingSelected = crossfilter.countSelected() === crossfilter.size();
+    const nothingColoredBy = !colorAccessor;
+    const noGenes = userDefinedGenes.length === 0 && diffexpGenes.length === 0;
+    const scatterNotDpl = !scatterplotXXaccessor || !scatterplotYYaccessor;
+    const nothingInCellsets = !celllist1 && !celllist2;
+
+    return (
+      nothingSelected &&
+      World.worldEqUniverse(world, universe) &&
+      nothingColoredBy &&
+      noGenes &&
+      scatterNotDpl &&
+      nothingInCellsets
+    );
+  };
+
+  resetInterface = () => {
+    const { dispatch } = this.props;
+    dispatch({
+      type: "interface reset started"
+    });
+    dispatch(actions.resetInterface());
+  };
+
   reglDraw(regl, drawPoints, sizeBuffer, colorBuffer, pointBuffer, camera) {
     regl.clear({
       depth: 1,
@@ -343,12 +394,21 @@ class Graph extends React.Component {
 
   // when a lasso is completed, filter to the points within the lasso polygon
   handleLassoEnd(polygon) {
+    const minimumPolygoneArea = 10;
     const { dispatch } = this.props;
 
-    dispatch({
-      type: "lasso selection",
-      polygon: polygon.map(xy => this.invertPoint(xy)) // transform the polygon
-    });
+    if (
+      polygon.length < 3 ||
+      Math.abs(d3.polygonArea(polygon)) < minimumPolygoneArea
+    ) {
+      // if less than three points, or super small area, treat as a clear selection.
+      dispatch({ type: "lasso deselect" });
+    } else {
+      dispatch({
+        type: "lasso selection",
+        polygon: polygon.map(xy => this.invertPoint(xy)) // transform the polygon
+      });
+    }
   }
 
   handleOpacityRangeChange(e) {
@@ -359,22 +419,16 @@ class Graph extends React.Component {
     });
   }
 
-  resetInterface() {
-    const { dispatch } = this.props;
-    dispatch({
-      type: "interface reset started"
-    });
-    dispatch(actions.resetInterface());
-  }
-
   render() {
     const {
       dispatch,
       responsive,
       crossfilter,
-      resettingInterface
+      resettingInterface,
+      library_versions,
+      undoDisabled,
+      redoDisabled
     } = this.props;
-
     const { mode } = this.state;
     return (
       <div id="graphWrapper">
@@ -399,10 +453,11 @@ class Graph extends React.Component {
             >
               <AnchorButton
                 type="button"
+                data-testid="subset-button"
                 disabled={
                   crossfilter &&
-                  (crossfilter.countFiltered() === 0 ||
-                    crossfilter.countFiltered() === crossfilter.size())
+                  (crossfilter.countSelected() === 0 ||
+                    crossfilter.countSelected() === crossfilter.size())
                 }
                 style={{ marginRight: 10 }}
                 onClick={() => {
@@ -418,15 +473,14 @@ class Graph extends React.Component {
               position="left"
             >
               <AnchorButton
-                disabled={
-                  false
-                  /* world && universe ? worldEqUniverse(world, universe) : false */
-                }
+                disabled={this.isResetDisabled()}
                 type="button"
                 loading={resettingInterface}
                 intent="warning"
                 style={{ marginRight: 10 }}
-                onClick={this.resetInterface.bind(this)}
+                onClick={this.resetInterface}
+                data-testid="reset"
+                data-testclass={`resetting-${resettingInterface}`}
               >
                 reset
               </AnchorButton>
@@ -436,6 +490,7 @@ class Graph extends React.Component {
                 <Tooltip content="Lasso selection" position="left">
                   <Button
                     type="button"
+                    data-testid="mode-lasso"
                     className="bp3-button bp3-icon-polygon-filter"
                     active={mode === "lasso"}
                     onClick={() => {
@@ -451,6 +506,7 @@ class Graph extends React.Component {
                 <Tooltip content="Pan and zoom" position="left">
                   <Button
                     type="button"
+                    data-testid="mode-pan-zoom"
                     className="bp3-button bp3-icon-zoom-in"
                     active={mode === "zoom"}
                     onClick={() => {
@@ -463,8 +519,84 @@ class Graph extends React.Component {
                     }}
                   />
                 </Tooltip>
+                <Tooltip content="Undo" position="left">
+                  <AnchorButton
+                    type="button"
+                    className="bp3-button bp3-icon-undo"
+                    disabled={undoDisabled}
+                    onClick={() => {
+                      dispatch({ type: "@@undoable/undo" });
+                    }}
+                    style={{
+                      cursor: "pointer"
+                    }}
+                  />
+                </Tooltip>
+                <Tooltip content="Redo" position="left">
+                  <AnchorButton
+                    type="button"
+                    className="bp3-button bp3-icon-redo"
+                    disabled={redoDisabled}
+                    onClick={() => {
+                      dispatch({ type: "@@undoable/redo" });
+                    }}
+                    style={{
+                      cursor: "pointer"
+                    }}
+                  />
+                </Tooltip>
               </div>
             </div>
+            <div style={{ marginLeft: 10 }}>
+              <Popover
+                content={
+                  <Menu>
+                    <MenuItem
+                      href="https://chanzuckerberg.github.io/cellxgene/faq.html"
+                      target="_blank"
+                      icon="help"
+                      text="FAQ"
+                    />
+                    <MenuItem
+                      href="https://join-cziscience-slack.herokuapp.com/"
+                      target="_blank"
+                      icon="chat"
+                      text="Chat"
+                    />
+                    <MenuItem
+                      href="https://chanzuckerberg.github.io/cellxgene/"
+                      target="_blank"
+                      icon="book"
+                      text="Docs"
+                    />
+                    <MenuItem
+                      href="https://github.com/chanzuckerberg/cellxgene"
+                      target="_blank"
+                      icon="git-branch"
+                      text="Github"
+                    />
+                    <MenuItem
+                      target="_blank"
+                      text={`cellxgene v${
+                        library_versions && library_versions.cellxgene
+                          ? library_versions.cellxgene
+                          : null
+                      }`}
+                    />
+                    <MenuItem text="MIT License" />
+                  </Menu>
+                }
+                position={Position.BOTTOM_RIGHT}
+              >
+                <Button
+                  type="button"
+                  className="bp3-button bp3-icon-cog"
+                  style={{
+                    cursor: "pointer"
+                  }}
+                />
+              </Popover>
+            </div>
           </div>
         </div>
         <div
@@ -485,6 +617,7 @@ class Graph extends React.Component {
             <canvas
               width={responsive.width - this.graphPaddingRight}
               height={responsive.height - this.graphPaddingTop}
+              data-testid="layout-graph"
               ref={canvas => {
                 this.reglCanvas = canvas;
               }}

client/src/components/graph/setupSVGandBrush.js

@@ -20,6 +20,7 @@ export default (
   const svg = d3
     .select("#graphAttachPoint")
     .append("svg")
+    .attr("data-testid", "layout-overlay")
     .attr("width", responsive.width - graphPaddingRight)
     .attr("height", responsive.height)
     .attr("class", `${styles.graphSVG}`);

client/src/components/leftsidebar.js

@@ -40,6 +40,7 @@ class LeftSideBar extends React.Component {
         }}
       >
         <p
+          data-testid="header"
           style={{
             position: "fixed",
             top: globals.cellxgeneTitleTopPadding,

client/src/components/scatterplot/scatterplot.js

@@ -19,31 +19,30 @@ import _drawPoints from "./drawPointsRegl";
 import scaleLinear from "../../util/scaleLinear";
 
 import { margin, width, height } from "./util";
-import { kvCache } from "../../util/stateManager";
 import finiteExtent from "../../util/finiteExtent";
 
 @connect(state => {
-  const {
-    world,
-    crossfilter,
-    scatterplotXXaccessor,
-    scatterplotYYaccessor
-  } = state.controls;
+  const { world, crossfilter } = state;
+  const { scatterplotXXaccessor, scatterplotYYaccessor } = state.controls;
   const expressionX =
-    world && scatterplotXXaccessor
-      ? kvCache.get(world.varDataCache, scatterplotXXaccessor)
+    world &&
+    scatterplotXXaccessor &&
+    world.varData.hasCol(scatterplotXXaccessor)
+      ? world.varData.col(scatterplotXXaccessor).asArray()
       : null;
   const expressionY =
-    world && scatterplotYYaccessor
-      ? kvCache.get(world.varDataCache, scatterplotYYaccessor)
+    world &&
+    scatterplotYYaccessor &&
+    world.varData.hasCol(scatterplotYYaccessor)
+      ? world.varData.col(scatterplotYYaccessor).asArray()
       : null;
 
   return {
     world,
 
-    colorRGB: state.controls.colorRGB,
-    colorAccessor: state.controls.colorAccessor,
-    colorScale: state.controls.colorScale,
+    colorRGB: state.colors.rgb,
+    colorScale: state.colors.scale,
+    colorAccessor: state.colors.colorAccessor,
 
     // Accessors are var/gene names (strings)
     scatterplotXXaccessor,
@@ -55,9 +54,7 @@ import finiteExtent from "../../util/finiteExtent";
     expressionX,
     expressionY,
 
-    crossfilter,
-    // updated whenever the crossfilter selection is updated
-    selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null)
+    crossfilter
   };
 })
 class Scatterplot extends React.Component {
@@ -65,12 +62,17 @@ class Scatterplot extends React.Component {
     super(props);
     this.count = 0;
     this.axes = false;
-    this.state = {
-      svg: null,
-      minimized: null,
+    this.renderCache = {
+      positions: null,
+      colors: null,
+      sizes: null,
       xScale: null,
       yScale: null
     };
+    this.state = {
+      svg: null,
+      minimized: null
+    };
   }
 
   componentDidMount() {
@@ -81,6 +83,7 @@ class Scatterplot extends React.Component {
     if (svg && expressionX && expressionY) {
       scales = Scatterplot.setupScales(expressionX, expressionY);
       this.drawAxesSVG(scales.xScale, scales.yScale, svg);
+      this.renderCache = { ...this.renderCache, ...scales };
     }
 
     const camera = _camera(this.reglCanvas, { scale: true, rotate: false });
@@ -113,8 +116,6 @@ class Scatterplot extends React.Component {
       pointBuffer,
       colorBuffer,
       svg,
-      xScale: scales ? scales.xScale : null,
-      yScale: scales ? scales.yScale : null,
       reglRender,
       camera,
       drawPoints
@@ -133,8 +134,6 @@ class Scatterplot extends React.Component {
     } = this.props;
     const {
       reglRender,
-      xScale,
-      yScale,
       regl,
       pointBuffer,
       colorBuffer,
@@ -145,17 +144,12 @@ class Scatterplot extends React.Component {
     } = this.state;
 
     if (
-      world &&
-      svg &&
-      xScale &&
-      yScale &&
-      scatterplotXXaccessor &&
-      scatterplotYYaccessor &&
-      (scatterplotXXaccessor !== prevProps.scatterplotXXaccessor || // was CLU now FTH1 etc
-      scatterplotYYaccessor !== prevProps.scatterplotYYaccessor || // was CLU now FTH1 etc
-        !this.axes) // clicked off the tab and back again, rerender
+      scatterplotXXaccessor !== prevProps.scatterplotXXaccessor || // was CLU now FTH1 etc
+      scatterplotYYaccessor !== prevProps.scatterplotYYaccessor // was CLU now FTH1 etc
     ) {
-      this.drawAxesSVG(xScale, yScale, svg);
+      const scales = Scatterplot.setupScales(expressionX, expressionY);
+      this.drawAxesSVG(scales.xScale, scales.yScale, svg);
+      this.renderCache = { ...this.renderCache, ...scales };
     }
 
     if (reglRender && this.reglRenderState === "rendering") {
@@ -172,35 +166,51 @@ class Scatterplot extends React.Component {
       expressionX &&
       expressionY &&
       scatterplotXXaccessor &&
-      scatterplotYYaccessor &&
-      xScale &&
-      yScale
+      scatterplotYYaccessor
     ) {
+      const { renderCache } = this;
+      const { xScale, yScale } = this.renderCache;
       const cellCount = expressionX.length;
-      const positionsBuf = new Float32Array(2 * cellCount);
-      const colorsBuf = new Float32Array(3 * cellCount);
-      const sizesBuf = new Float32Array(cellCount);
 
-      const glScaleX = scaleLinear([0, width], [-0.95, 0.95]);
-      const glScaleY = scaleLinear([0, height], [-1, 1]);
-
-      /*
-        Construct Vectors
-      */
-      for (let i = 0; i < cellCount; i += 1) {
-        positionsBuf[2 * i] = glScaleX(xScale(expressionX[i]));
-        positionsBuf[2 * i + 1] = glScaleY(yScale(expressionY[i]));
+      // Points change when expressionX or expressionY change.
+      if (
+        !renderCache.positions ||
+        expressionX !== prevProps.expressionX ||
+        expressionY !== prevProps.expressionY
+      ) {
+        if (!renderCache.positions) {
+          renderCache.positions = new Float32Array(2 * cellCount);
+        }
+        const glScaleX = scaleLinear([0, width], [-0.95, 0.95]);
+        const glScaleY = scaleLinear([0, height], [-1, 1]);
+        for (let i = 0, { positions } = renderCache; i < cellCount; i += 1) {
+          positions[2 * i] = glScaleX(xScale(expressionX[i]));
+          positions[2 * i + 1] = glScaleY(yScale(expressionY[i]));
+        }
+        pointBuffer({ data: renderCache.positions, dimension: 2 });
       }
 
-      for (let i = 0; i < cellCount; i += 1) {
-        colorsBuf.set(colorRGB[i], 3 * i);
+      // Colors for each point - change only when props.colorsRGB change.
+      if (!renderCache.colors || colorRGB !== prevProps.colorRGB) {
+        if (!renderCache.colors) {
+          renderCache.colors = new Float32Array(3 * cellCount);
+        }
+        for (let i = 0, { colors } = renderCache; i < cellCount; i += 1) {
+          colors.set(colorRGB[i], 3 * i);
+        }
+        colorBuffer({ data: renderCache.colors, dimension: 3 });
       }
 
-      crossfilter.fillByIsFiltered(sizesBuf, 4, 0.2);
+      // Sizes for each point - updates are triggered only when selected
+      // obs change
+      if (!renderCache.sizes || crossfilter !== prevProps.crossfilter) {
+        if (!renderCache.sizes) {
+          renderCache.sizes = new Float32Array(cellCount);
+        }
+        crossfilter.fillByIsSelected(renderCache.sizes, 4, 0.2);
+        sizeBuffer({ data: renderCache.sizes, dimension: 1 });
+      }
 
-      pointBuffer({ data: positionsBuf, dimension: 2 });
-      colorBuffer({ data: colorsBuf, dimension: 3 });
-      sizeBuffer({ data: sizesBuf, dimension: 1 });
       this.count = cellCount;
 
       regl._refresh();
@@ -213,16 +223,6 @@ class Scatterplot extends React.Component {
         camera
       );
     }
-
-    if (
-      expressionX &&
-      expressionY &&
-      (scatterplotXXaccessor !== prevProps.scatterplotXXaccessor || // was CLU now FTH1 etc
-        scatterplotYYaccessor !== prevProps.scatterplotYYaccessor)
-    ) {
-      const scales = Scatterplot.setupScales(expressionX, expressionY);
-      this.setState(scales);
-    }
   }
 
   static setupScales(expressionX, expressionY) {
@@ -338,14 +338,12 @@ class Scatterplot extends React.Component {
           <Button
             type="button"
             minimal
-            onClick={() => {
+            data-testid="clear-scatterplot"
+            onClick={() =>
               dispatch({
                 type: "clear scatterplot"
-              });
-              dispatch({
-                type: "reset colorscale"
-              });
-            }}
+              })
+            }
           >
             remove
           </Button>
@@ -361,6 +359,7 @@ class Scatterplot extends React.Component {
           <canvas
             width={width}
             height={height}
+            data-testid="scatterplot"
             style={{
               marginLeft: margin.left - 7,
               marginTop: margin.top

client/src/components/scatterplot/setupScatterplot.js

@@ -14,6 +14,7 @@ const setupScatterplot = (width, height, margin) => {
     .append("svg")
     .attr("width", width + margin.left + margin.right)
     .attr("height", height + margin.top + margin.bottom)
+    .attr("data-testid", "scatterplot-svg")
     .append("g")
     .attr("transform", `translate(${margin.left},${margin.top})`);
 

client/src/globals.js

@@ -1,35 +1,5 @@
 import { Colors } from "@blueprintjs/core";
 
-// jshint esversion: 6
-/* these will be either (preferably) specified or inferred */
-export const categories = [
-  "Sample.type",
-  "Selection",
-  "Location",
-  "Sample.name",
-  "Class",
-  "Neoplastic"
-];
-export const continuous = [
-  "Total_reads",
-  "Unique_reads",
-  "Unique_reads_percent",
-  "ERCC_reads",
-  "Non_ERCC_reads",
-  "ERCC_to_non_ERCC",
-  "Genes_detected",
-  "Multimapping_reads_percent",
-  "Splice_sites_AT.AC",
-  "Splice_sites_Annotated",
-  "Splice_sites_GC.AG",
-  "Splice_sites_GT.AG",
-  "Splice_sites_non_canonical",
-  "Splice_sites_total",
-  "Unmapped_mismatch",
-  "Unmapped_other",
-  "Unmapped_short"
-];
-
 /* if a categorical metadata field has more options than this, truncate */
 export const maxCategoricalOptionsToDisplay = 100;
 

client/src/middleware/updateCellColors.js

@@ -1,149 +0,0 @@
-// jshint esversion: 6
-import _ from "lodash";
-import * as d3 from "d3";
-import { interpolateRainbow, interpolateCool } from "d3-scale-chromatic";
-import * as globals from "../globals";
-import parseRGB from "../util/parseRGB";
-import finiteExtent from "../util/finiteExtent";
-
-/*
-  https://medium.com/@jacobp100/you-arent-using-redux-middleware-enough-94ffe991e6
-  storeInstance =>
-    functionToCallWithAnActionThatWillSendItToTheNextMiddleware =>
-    actionThatDispatchWasCalledWith =>
-    valueToUseAsTheReturnValueOfTheDispatchCall
-*/
-
-/*
-  What this file does:
-
-  1. fire a filter action anywhere in the app
-  2. ** this middleware checks to see the state of all the currently selected filters,
-     including the new one
-  3. ** create updated selection from a copy of all the cells presently on the client
-     (this may be a subset of 'all')
-  4. ** append that new selection to the action so that it magically appears in the reducer
-     just because the action was fired
-
-  This is nice because we keep a lot of filtering business logic centralized
-  (what it means in practice to be selected)
-*/
-
-const updateCellColorsMiddleware = store => next => action => {
-  const s = store.getState();
-
-  /*
-  this is a hardcoded map of the things we need to keep an eye on and update
-  global cell selection in response to
-  */
-  const filterJustChanged =
-    action.type === "color by expression" ||
-    action.type === "color by continuous metadata" ||
-    action.type === "color by categorical metadata";
-
-  if (!filterJustChanged || !s.controls.world.obsAnnotations) {
-    return next(
-      action
-    ); /* if the cells haven't loaded or the action wasn't a color change, bail */
-  }
-
-  const { obsAnnotations } = s.controls.world;
-  let colorScale;
-  const colorsByRGB = new Array(obsAnnotations.length);
-
-  /*
-  in plain language...
-   (a) once the cells have loaded.
-   (b) each time a user changes a color control we need to update cellsMetadata colors
-  This is available to all the draw functions as controls.colorRGB[index]
-  */
-
-  if (action.type === "color by categorical metadata") {
-    const { categories } = _.filter(s.controls.world.schema.annotations.obs, {
-      name: action.colorAccessor
-    })[0];
-
-    colorScale = d3
-      .scaleSequential(interpolateRainbow)
-      .domain([0, categories.length]);
-
-    /* pre-create colors - much faster than doing it for each obs */
-    const colors = _.transform(categories, (acc, cat, idx) => {
-      acc[cat] = parseRGB(colorScale(idx));
-    });
-
-    const key = action.colorAccessor;
-    for (let i = 0, len = obsAnnotations.length; i < len; i += 1) {
-      const obs = obsAnnotations[i];
-      const cat = obs[key];
-      colorsByRGB[i] = colors[cat];
-    }
-  }
-
-  if (action.type === "color by continuous metadata") {
-    const colorBins = 100;
-    const [min, max] = [0, action.rangeMaxForColorAccessor];
-    colorScale = d3
-      .scaleQuantile()
-      .domain([min, max])
-      .range(_.range(colorBins - 1, -1, -1));
-
-    /* pre-create colors - much faster than doing it for each obs */
-    const colors = new Array(colorBins);
-    for (let i = 0; i < colorBins; i += 1) {
-      colors[i] = parseRGB(interpolateCool(i / colorBins));
-    }
-
-    const key = action.colorAccessor;
-    const nonFiniteColor = parseRGB(globals.nonFiniteCellColor);
-    for (let i = 0, len = obsAnnotations.length; i < len; i += 1) {
-      const val = obsAnnotations[i][key];
-      if (Number.isFinite(val)) {
-        const c = colorScale(val);
-        colorsByRGB[i] = colors[c];
-      } else {
-        colorsByRGB[i] = nonFiniteColor;
-      }
-    }
-  }
-
-  if (action.type === "color by expression") {
-    const { gene, data } = action;
-    const expression = data[gene]; // Float32Array
-    const colorBins = 100;
-    const [min, max] = finiteExtent(expression);
-    colorScale = d3
-      .scaleQuantile()
-      .domain([min, max])
-      .range(_.range(colorBins - 1, -1, -1));
-
-    /* pre-create colors - much faster than doing it for each obs */
-    const colors = new Array(colorBins);
-    for (let i = 0; i < colorBins; i += 1) {
-      colors[i] = parseRGB(interpolateCool(i / colorBins));
-    }
-    const nonFiniteColor = parseRGB(globals.nonFiniteCellColor);
-
-    for (let i = 0, len = expression.length; i < len; i += 1) {
-      const e = expression[i];
-      if (Number.isFinite(e)) {
-        const c = colorScale(e);
-        colorsByRGB[i] = colors[c];
-      } else {
-        colorsByRGB[i] = nonFiniteColor;
-      }
-    }
-  }
-
-  /*
-  append the result of all the filters to the action the user just triggered
-  */
-  const modifiedAction = Object.assign({}, action, {
-    colors: { rgb: colorsByRGB },
-    colorScale
-  });
-
-  return next(modifiedAction);
-};
-
-export default updateCellColorsMiddleware;

client/src/middleware/updateURLMiddleware.js

@@ -1,87 +0,0 @@
-// jshint esversion: 6
-// import uri from "urijs";
-
-/*
-NOTE: file currently not used, but retained as we expect to reinstate features in this
-area shortly.
-*/
-
-/*
-  https://medium.com/@jacobp100/you-arent-using-redux-middleware-enough-94ffe991e6
-  storeInstance
-    => functionToCallWithAnActionThatWillSendItToTheNextMiddleware
-    => actionThatDispatchWasCalledWith
-    => valueToUseAsTheReturnValueOfTheDispatchCall
-*/
-
-const updateURLMiddleware = (/* store */) => next => action => {
-  // const oldState = store.getState();
-  const nextAction = next(action);
-
-  if (action.type === "url changed") {
-    /* we don't handle pop state here - we handle it in the url reducer */
-    return nextAction;
-  }
-
-  // const state = store.getState();
-
-  /************************************************************************
-      *************************************************************************
-      1. Redux app state just changed. Clear URL, and then update it.
-      1a. We get the whole state tree to construct the url!
-      1b. But (see reducers/url.js) we try to centralize it because...
-      1c. ...the back button / initial load case ('url changed' return above)
-      means that we have to listen for 'url changed' and construct state
-      from the browser
-      *************************************************************************
-      ************************************************************************/
-
-  // const oldURI = URI(window.location.href)
-  // const newURI = URI(oldURI).setQuery({})
-
-  // if (window.location.search === "") {
-  //   newURL = uri.addQuery(category, value).toString(); /* #1 */
-  // } else if (uri.hasQuery(category, value) || uri.hasQuery(category, value, true)) { /* true param here means check arrays as well http://medialize.github.io/URI.js/docs.html#search-has */
-  //   newURL = uri.removeQuery(category, value).toString(); /* #4 */
-  // } else {
-  //   newURL = uri.addQuery(category, value).toString(); /* #2 & #3 are handled by URI */
-  // }
-  //
-  // window.history.pushState("", "", newURL)
-
-  //
-  // // Internal helper for working with URIs
-  // const oldURI = new URI(window.location.href);
-  // const newURI = new URI(oldURI).setQueryData({});
-  //
-  // newURI.setPath('/foo/bar');
-  //
-  // // Set the path based on state
-  // if (!state.isOnLandingPage && state.project.id) {
-  //   newURI.setPath(newURI.getPath() + state.project.id + '/');
-  //   newURI.addQueryData('baz', state.mode);
-  //   newURI.addQueryData('bat', state.selection.activePageID);
-  // } else {
-  //   newURI.setPath(newURI.getPath() + state.landingSection + '/');
-  // }
-  //
-  // // Avoid URL thrashing by replacing state while loading instead of pushing
-  // const newPath = newURI.toString();
-  // const oldPath = oldURI.toString();
-  // if (newPath !== oldPath) {
-  //   if (
-  //     (oldState.mode === 'asdf' &&
-  //       state.mode === 'asdf' &&
-  //       !oldState.isOnLandingPage) ||
-  //     oldState.isLoadingProject !== state.isLoadingProject
-  //   ) {
-  //     window.history.replaceState(null, null, newPath);
-  //   } else {
-  //     window.history.pushState(null, null, newPath);
-  //   }
-  // }
-
-  return nextAction;
-};
-
-export default updateURLMiddleware;

client/src/reducers/cascade.js

@@ -0,0 +1,44 @@
+export default function cascadeReducers(arg) {
+  /*
+  Combined a set of cascading reducers into a single reducer.  Cascading
+  reducers are reducers which may rely on state computed by another reducer.
+  Therefore, they:
+  - must be composed in a particular order (currently, this is a simple
+    linear list of reducers, run in list order)
+  - must have access to partially updated "next state" so they can further
+    derive state.
+
+  Parameter is one of:
+  - a Map object
+  - an array of tuples, [ [key1, reducer1], [key2, reducer2], ... ]
+  Ie,  cascadeReducers([ ["a", reduceA], ["b", reduceB] ])
+
+  Each reducer will be called with the sigature:
+      (prevState, action, sharedNextState, sharedPrevState) => newState
+
+  cascadeReducers will build a composite newState object, much
+  like combinedReducers.  Additional semantics:
+  - reducers guaranteed to be called in order
+  - each reducer will receive shared objects
+  */
+  const reducers = arg instanceof Map ? arg : new Map(arg);
+  const reducerKeys = [...reducers.keys()];
+  return (prevState, action) => {
+    const nextState = {};
+    let stateChange = false;
+    for (let i = 0, l = reducerKeys.length; i < l; i += 1) {
+      const key = reducerKeys[i];
+      const reducer = reducers.get(key);
+      const prevStateForKey = prevState ? prevState[key] : undefined;
+      const nextStateForKey = reducer(
+        prevStateForKey,
+        action,
+        nextState,
+        prevState
+      );
+      nextState[key] = nextStateForKey;
+      stateChange = stateChange || nextStateForKey !== prevStateForKey;
+    }
+    return stateChange ? nextState : prevState;
+  };
+}

client/src/reducers/categoricalSelection.js

@@ -0,0 +1,105 @@
+import _ from "lodash";
+
+import { ControlsHelpers } from "../util/stateManager";
+import * as globals from "../globals";
+
+function maxCategoryItems(state) {
+  return _.get(
+    state.config,
+    "parameters.max-category-items",
+    globals.configDefaults.parameters["max-category-items"]
+  );
+}
+
+const CategoricalSelection = (
+  state,
+  action,
+  nextSharedState,
+  prevSharedState
+) => {
+  switch (action.type) {
+    case "initial data load complete (universe exists)":
+    case "set World to current selection":
+    case "reset World to eq Universe": {
+      const { world } = nextSharedState;
+      return ControlsHelpers.createCategoricalSelection(
+        maxCategoryItems(prevSharedState),
+        world
+      );
+    }
+
+    case "categorical metadata filter select": {
+      /*
+      Set the specific category in this field to false
+      */
+      const newCategorySelected = Array.from(
+        state[action.metadataField].categorySelected
+      );
+      newCategorySelected[action.categoryIndex] = true;
+      const newCategoricalSelection = {
+        ...state,
+        [action.metadataField]: {
+          ...state[action.metadataField],
+          categorySelected: newCategorySelected
+        }
+      };
+      return newCategoricalSelection;
+    }
+
+    case "categorical metadata filter deselect": {
+      /*
+      Set the specific category in this field to false
+      */
+      const newCategorySelected = Array.from(
+        state[action.metadataField].categorySelected
+      );
+      newCategorySelected[action.categoryIndex] = false;
+      const newCategoricalSelection = {
+        ...state,
+        [action.metadataField]: {
+          ...state[action.metadataField],
+          categorySelected: newCategorySelected
+        }
+      };
+      return newCategoricalSelection;
+    }
+
+    case "categorical metadata filter none of these": {
+      /*
+      set all categories in this field to false.
+      */
+      const newCategoricalSelection = {
+        ...state,
+        [action.metadataField]: {
+          ...state[action.metadataField],
+          categorySelected: Array.from(
+            state[action.metadataField].categorySelected
+          ).fill(false)
+        }
+      };
+      return newCategoricalSelection;
+    }
+
+    case "categorical metadata filter all of these": {
+      /*
+      set all categories in this field to true.
+      */
+      const newCategoricalSelection = {
+        ...state,
+        [action.metadataField]: {
+          ...state[action.metadataField],
+          categorySelected: Array.from(
+            state[action.metadataField].categorySelected
+          ).fill(true)
+        }
+      };
+      return newCategoricalSelection;
+    }
+
+    default: {
+      return state;
+    }
+  }
+};
+
+export default CategoricalSelection;

client/src/reducers/colors.js

@@ -0,0 +1,88 @@
+import { createColors } from "../util/stateManager";
+
+const ColorsReducer = (
+  state = {
+    colorMode: null,
+    colorAccessor: null,
+    rgb: null,
+    scale: null
+  },
+  action,
+  nextSharedState,
+  prevSharedState
+) => {
+  switch (action.type) {
+    case "initial data load complete (universe exists)":
+    case "reset World to eq Universe": {
+      const { world } = nextSharedState;
+      const colorMode = null;
+      const colorAccessor = null;
+      const { rgb, scale } = createColors(world, colorMode);
+      return {
+        ...state,
+        colorAccessor,
+        colorMode,
+        rgb,
+        scale
+      };
+    }
+
+    case "set World to current selection": {
+      const { colorMode, colorAccessor } = state;
+      const { world } = nextSharedState;
+      const { rgb, scale } = createColors(world, colorMode, colorAccessor);
+      return {
+        ...state,
+        rgb,
+        scale
+      };
+    }
+
+    case "reset colorscale": {
+      const { world } = prevSharedState;
+      const { rgb, scale } = createColors(world);
+      return {
+        ...state,
+        colorMode: null,
+        colorAccessor: null,
+        rgb,
+        scale
+      };
+    }
+
+    case "color by categorical metadata":
+    case "color by continuous metadata": {
+      const { world } = prevSharedState;
+      const { rgb, scale } = createColors(
+        world,
+        action.type,
+        action.colorAccessor
+      );
+      return {
+        ...state,
+        colorMode: action.type,
+        colorAccessor: action.colorAccessor,
+        rgb,
+        scale
+      };
+    }
+
+    case "color by expression": {
+      const { world } = prevSharedState;
+      const { rgb, scale } = createColors(world, action.type, action.gene);
+      return {
+        ...state,
+        colorMode: action.type,
+        colorAccessor: action.gene,
+        rgb,
+        scale
+      };
+    }
+
+    default: {
+      return state;
+    }
+  }
+};
+
+export default ColorsReducer;

client/src/reducers/continuousSelection.js

@@ -0,0 +1,26 @@
+import { makeContinuousDimensionName } from "../util/nameCreators";
+
+const ContinuousSelection = (state = {}, action) => {
+  switch (action.type) {
+    case "reset World to eq Universe": {
+      return {};
+    }
+    case "continuous metadata histogram start":
+    case "continuous metadata histogram brush":
+    case "continuous metadata histogram end": {
+      const name = makeContinuousDimensionName(
+        action.continuousNamespace,
+        action.selection
+      );
+      return {
+        ...state,
+        [name]: action.range
+      };
+    }
+    default: {
+      return state;
+    }
+  }
+};
+
+export default ContinuousSelection;

client/src/reducers/controls.js

@@ -1,104 +1,8 @@
 // jshint esversion: 6
 
 import _ from "lodash";
-import { polygonContains } from "d3";
 
-import { World, kvCache, WorldUtil } from "../util/stateManager";
-import parseRGB from "../util/parseRGB";
-import Crossfilter from "../util/typedCrossfilter";
-import * as globals from "../globals";
-import {
-  layoutDimensionName,
-  obsAnnoDimensionName,
-  userDefinedDimensionName,
-  diffexpDimensionName,
-  makeContinuousDimensionName
-} from "../util/nameCreators";
-import { fillRange } from "../util/typedCrossfilter/util";
-
-/*
-Selection state for categoricals are tracked in an Object that
-has two main components for each category:
-1. mapping of option value to an index
-2. array of bool selection state by index
-Remember that option values can be ANY js type, except undefined/null.
-
-  {
-    _category_name_1: {
-      // map of option value to index
-      categoryIndices: Map([
-        catval1: index,
-        ...
-      ])
-
-      // index->selection true/false state
-      categorySelected: [ true/false, true/false, ... ]
-
-      // number of options
-      numCategories: number,
-
-      // isTruncated - true if the options for selection has
-      // been truncated (ie, was too large to implement)
-    }
-  }
-*/
-function topNCategories(summary) {
-  const counts = _.map(summary.categories, cat =>
-    summary.categoryCounts.get(cat)
-  );
-  const sortIndex = fillRange(new Array(summary.numCategories)).sort(
-    (a, b) => counts[b] - counts[a]
-  );
-  const sortedCategories = _.map(sortIndex, i => summary.categories[i]);
-  const sortedCounts = _.map(sortIndex, i => counts[i]);
-  const N = globals.maxCategoricalOptionsToDisplay;
-
-  if (sortedCategories.length < N) {
-    return [sortedCategories, sortedCounts];
-  }
-  return [sortedCategories.slice(0, N), sortedCounts.slice(0, N)];
-}
-
-function createCategoricalSelectionState(state, world) {
-  const res = {};
-  _.forEach(world.summary.obs, (value, key) => {
-    if (value.categories) {
-      const isColorField = key.includes("color") || key.includes("Color");
-      const isSelectableCategory =
-        !isColorField &&
-        key !== "name" &&
-        value.categories.length < state.maxCategoryItems;
-      if (isSelectableCategory) {
-        const [categoryValues, categoryCounts] = topNCategories(value);
-        const categoryIndices = new Map(categoryValues.map((v, i) => [v, i]));
-        const numCategories = categoryIndices.size;
-        const categorySelected = new Array(numCategories).fill(true);
-        const isTruncated = categoryValues.length < value.numCategories;
-        res[key] = {
-          categoryValues, // array: of natively typed category values
-          categoryIndices, // map: category value (native type) -> category index
-          categorySelected, // array: t/f selection state
-          numCategories, // number: of categories
-          isTruncated, // bool: true if list was truncated
-          categoryCounts // array: cardinality of each category
-        };
-      }
-    }
-  });
-  return res;
-}
-
-/*
-given a categoricalSelectionState, return the list of all category values
-where selection state is true (ie, they are selected).
-*/
-function selectedValuesForCategory(categorySelectionState) {
-  const selectedValues = _([...categorySelectionState.categoryIndices])
-    .filter(tuple => categorySelectionState.categorySelected[tuple[1]])
-    .map(tuple => tuple[0])
-    .value();
-  return selectedValues;
-}
+import { WorldUtil } from "../util/stateManager";
 
 const Controls = (
   state = {
@@ -106,37 +10,23 @@ const Controls = (
     loading: false,
     error: null,
 
-    // configuration
-    maxCategoryItems: globals.configDefaults.parameters["max-category-items"],
-
-    // the whole big bang
-    universe: null,
-
     // all of the data + selection state
-    world: null,
-    colorRGB: null,
-    categoricalSelectionState: null,
-    crossfilter: null,
-    dimensionMap: null,
     userDefinedGenes: [],
     userDefinedGenesLoading: false,
     diffexpGenes: [],
 
-    colorAccessor: null,
-    colorScale: null,
     resettingInterface: false,
 
     opacityForDeselectedCells: 0.2,
-    graphBrushSelection: null,
-    continuousSelection: null,
     scatterplotXXaccessor: null, // just easier to read
     scatterplotYYaccessor: null,
-    axesHaveBeenDrawn: false,
     graphRenderCounter: 0 /* integer as <Component key={graphRenderCounter} - a change in key forces a remount */,
     __storedStateForCelllist1__: null /* will need procedural control of brush ie., brush.extent https://bl.ocks.org/micahstubbs/3cda05ca68cba260cb81 */,
     __storedStateForCelllist2__: null
   },
-  action
+  action,
+  nextSharedState,
+  prevSharedState
 ) => {
   /*
   For now, log anything looking like an error to the console.
@@ -150,182 +40,32 @@ const Controls = (
           Initialization, World/Universe management
           and data loading.
     ******************************************************/
-    case "configuration load complete": {
-      // there are a couple of configuration items we need to retain
-      return {
-        ...state,
-        maxCategoryItems: _.get(
-          state.config,
-          "parameters.max-category-items",
-          globals.configDefaults.parameters["max-category-items"]
-        )
-      };
-    }
     case "initial data load start": {
       return { ...state, loading: true };
     }
-    case "initial data load complete (universe exists)":
-    case "reset World to eq Universe": {
-      const { userDefinedGenes, diffexpGenes } = state;
+    case "initial data load complete (universe exists)": {
       /* first light - create world & other data-driven defaults */
-      const { universe } = action;
-      const world = World.createWorldFromEntireUniverse(universe);
-      const colorRGB = new Array(universe.nObs).fill(
-        parseRGB(globals.defaultCellColor)
-      );
-      const categoricalSelectionState = createCategoricalSelectionState(
-        state,
-        world
-      );
-      const crossfilter = Crossfilter(world.obsAnnotations);
-      const dimensionMap = World.createObsDimensionMap(crossfilter, world);
       WorldUtil.clearCaches();
-
-      const worldVarDataCache = world.varDataCache;
-
-      // dimensionMap = {
-      //     layout_X: dim-for-X,
-      //     obsAnno_name: dim for an annotation,
-      //     varData_userDefined_genename: dim for user defined expression,
-      //     varData_diffexp_genename: dim for diff-exp added gene expression
-      // }
-      /* var dimensions */
-      if (userDefinedGenes.length > 0) {
-        /*
-          verbose & slightly confusing that we also access this as an object
-          in controls rather than an array, should be abstracted into
-          util ie., createDimensionsFromBothListsOfGenes(userGenes, diffExp)
-        */
-        _.forEach(userDefinedGenes, gene => {
-          dimensionMap[
-            userDefinedDimensionName(gene)
-          ] = World.createVarDimension(
-            /* "__var__" + */
-            world,
-            worldVarDataCache,
-            crossfilter,
-            gene
-          );
-        });
-      }
-
-      if (diffexpGenes.length > 0) {
-        _.forEach(diffexpGenes, gene => {
-          dimensionMap[diffexpDimensionName(gene)] = World.createVarDimension(
-            /* "__var__" + */
-            world,
-            worldVarDataCache,
-            crossfilter,
-            gene
-          );
-        });
-      }
-
       return {
         ...state,
         loading: false,
         error: null,
-        universe,
-        world,
-        colorRGB,
-        categoricalSelectionState,
-        crossfilter,
-        dimensionMap,
-        colorAccessor: null,
+        resettingInterface: false
+      };
+    }
+    case "reset World to eq Universe": {
+      WorldUtil.clearCaches();
+      return {
+        ...state,
         resettingInterface: false
       };
     }
     case "set World to current selection": {
-      const { userDefinedGenes, diffexpGenes } = state;
-
-      /* Set viewable world to be the currently selected data */
-      const world = World.createWorldFromCurrentSelection(
-        action.universe,
-        action.world,
-        action.crossfilter
-      );
-      const colorRGB = new Array(world.nObs).fill(
-        parseRGB(globals.defaultCellColor)
-      );
-      const categoricalSelectionState = createCategoricalSelectionState(
-        state,
-        world
-      );
-      const crossfilter = Crossfilter(world.obsAnnotations);
-      const dimensionMap = World.createObsDimensionMap(crossfilter, world);
       WorldUtil.clearCaches();
-
-      const worldVarDataCache = world.varDataCache;
-      /* var dimensions */
-
-      if (userDefinedGenes.length > 0) {
-        /*
-          verbose & slightly confusing that we also access this as an object
-          in controls rather than an array, should be abstracted into
-          util ie., createDimensionsFromBothListsOfGenes(userGenes, diffExp)
-        */
-        _.forEach(userDefinedGenes, gene => {
-          dimensionMap[
-            userDefinedDimensionName(gene)
-          ] = World.createVarDimension(
-            /* "__var__" + */
-            world,
-            worldVarDataCache,
-            crossfilter,
-            gene
-          );
-        });
-      }
-
-      if (diffexpGenes.length > 0) {
-        _.forEach(diffexpGenes, gene => {
-          dimensionMap[diffexpDimensionName(gene)] = World.createVarDimension(
-            /* "__var__" + */
-            world,
-            worldVarDataCache,
-            crossfilter,
-            gene
-          );
-        });
-      }
-
       return {
         ...state,
         loading: false,
-        error: null,
-        world,
-        colorRGB,
-        categoricalSelectionState,
-        crossfilter,
-        dimensionMap,
-        colorAccessor: null
-      };
-    }
-    case "expression load success": {
-      const { world, universe } = state;
-      let universeVarDataCache = universe.varDataCache;
-      let worldVarDataCache = world.varDataCache;
-      _.forEach(action.expressionData, (val, key) => {
-        universeVarDataCache = kvCache.set(universeVarDataCache, key, val);
-        if (kvCache.get(worldVarDataCache, key) === undefined) {
-          worldVarDataCache = kvCache.set(
-            worldVarDataCache,
-            key,
-            World.subsetVarData(world, universe, val)
-          );
-        }
-      });
-
-      return {
-        ...state,
-        universe: {
-          ...universe,
-          varDataCache: universeVarDataCache
-        },
-        world: {
-          ...world,
-          varDataCache: worldVarDataCache
-        }
+        error: null
       };
     }
     case "request user defined gene started": {
@@ -341,133 +81,50 @@ const Controls = (
       };
     }
     case "request user defined gene success": {
-      const { world, crossfilter, dimensionMap, userDefinedGenes } = state;
-      const worldVarDataCache = world.varDataCache;
-      const _userDefinedGenes = userDefinedGenes.slice();
-      const gene = action.data.genes[0];
-
-      dimensionMap[userDefinedDimensionName(gene)] = World.createVarDimension(
-        /* "__var__" + */
-        world,
-        worldVarDataCache,
-        crossfilter,
-        gene
+      const { userDefinedGenes } = state;
+      const _userDefinedGenes = _.uniq(
+        userDefinedGenes.concat(action.data.genes)
       );
-
       return {
         ...state,
-        dimensionMap,
         userDefinedGenes: _userDefinedGenes,
         userDefinedGenesLoading: false
       };
     }
     case "request differential expression success": {
-      const { world, crossfilter, dimensionMap } = state;
-      const worldVarDataCache = world.varDataCache;
+      const { world } = prevSharedState;
       const _diffexpGenes = [];
-
       action.data.forEach(d => {
-        _diffexpGenes.push(world.varAnnotations[d[0]].name);
+        _diffexpGenes.push(world.varAnnotations.at(d[0], "name"));
       });
-
-      _.forEach(_diffexpGenes, gene => {
-        dimensionMap[diffexpDimensionName(gene)] = World.createVarDimension(
-          /* "__var__" + */
-          world,
-          worldVarDataCache,
-          crossfilter,
-          gene
-        );
-      });
-
       return {
         ...state,
-        dimensionMap,
         diffexpGenes: _diffexpGenes
       };
     }
     case "clear differential expression": {
-      const { world, universe, dimensionMap } = state;
-      const _dimensionMap = dimensionMap;
-      const universeVarDataCache = universe.varDataCache;
-      const worldVarDataCache = world.varDataCache;
-
-      _.forEach(action.diffExp, values => {
-        const { name } = world.varAnnotations[values[0]];
-        // clean up crossfilter dimensions
-        const dimension = dimensionMap[diffexpDimensionName(name)];
-        dimension.dispose();
-        delete dimensionMap[diffexpDimensionName(name)];
-      });
       return {
         ...state,
-        dimensionMap: _dimensionMap,
-        diffexpGenes: [],
-        universe: {
-          ...universe,
-          varDataCache: universeVarDataCache
-        },
-        world: {
-          ...world,
-          varDataCache: worldVarDataCache
-        }
-      };
-    }
-    case "user defined gene": {
-      /*
-        this could also live in expression success with a conditional,
-        but that handles diffexp also
-      */
-      const newUserDefinedGenes = state.userDefinedGenes.slice();
-      newUserDefinedGenes.push(action.data);
-      return {
-        ...state,
-        userDefinedGenes: newUserDefinedGenes
+        diffexpGenes: []
       };
     }
     case "clear user defined gene": {
-      const { userDefinedGenes, dimensionMap } = state;
+      const { userDefinedGenes } = state;
       const newUserDefinedGenes = _.filter(
         userDefinedGenes,
         d => d !== action.data
       );
-
-      const dimension = dimensionMap[userDefinedDimensionName(action.data)];
-      dimension.dispose();
-      delete dimensionMap[userDefinedDimensionName(action.data)];
-
       return {
         ...state,
-        dimensionMap,
         userDefinedGenes: newUserDefinedGenes
       };
     }
     case "clear all user defined genes": {
-      const { userDefinedGenes, dimensionMap } = state;
-
-      _.forEach(userDefinedGenes, gene => {
-        const dimension = dimensionMap[userDefinedDimensionName(gene)];
-        dimension.dispose();
-        delete dimensionMap[userDefinedDimensionName(gene)];
-      });
-
       return {
         ...state,
-        dimensionMap,
         userDefinedGenes: []
       };
     }
-    case "reset colorscale": {
-      const { world } = state;
-      const colorRGB = new Array(world.nObs).fill(
-        parseRGB(globals.defaultCellColor)
-      );
-      return {
-        ...state,
-        colorRGB,
-        colorAccessor: null
-      };
-    }
     case "expression load error":
     case "initial data load error": {
       return {
@@ -480,52 +137,6 @@ const Controls = (
     /*******************************
              User Events
      *******************************/
-    case "graph brush selection change": {
-      state.dimensionMap[layoutDimensionName("XY")].filterWithinRect(
-        action.brushCoords.northwest,
-        action.brushCoords.southeast
-      );
-      return {
-        ...state,
-        graphBrushSelection: action.brushCoords
-      };
-    }
-    case "lasso deselect":
-    case "graph brush deselect": {
-      state.dimensionMap[layoutDimensionName("XY")].filterAll();
-      return {
-        ...state,
-        graphBrushSelection: null
-      };
-    }
-    case "lasso selection": {
-      const { polygon } = action;
-      const dXY = state.dimensionMap[layoutDimensionName("XY")];
-      if (polygon.length < 3) {
-        // single point or a line is not a polygon, and is therefore a deselect
-        dXY.filterAll();
-      } else {
-        dXY.filterWithinPolygon(polygon);
-      }
-      return {
-        ...state
-      };
-    }
-    case "continuous metadata histogram brush": {
-      const name = makeContinuousDimensionName(
-        action.continuousNamespace,
-        action.selection
-      );
-
-      // action.selection: metadata name being selected
-      // action.range: filter range, or null if deselected
-      if (!action.range) {
-        state.dimensionMap[name].filterAll();
-      } else {
-        state.dimensionMap[name].filterRange(action.range);
-      }
-      return { ...state };
-    }
     case "change opacity deselected cells in 2d graph background":
       return {
         ...state,
@@ -544,116 +155,6 @@ const Controls = (
         resettingInterface: true
       };
     }
-    /*******************************
-          Categorical metadata
-    *******************************/
-    case "categorical metadata filter select": {
-      const newCategorySelected = Array.from(
-        state.categoricalSelectionState[action.metadataField].categorySelected
-      );
-      newCategorySelected[action.categoryIndex] = true;
-      const newCategoricalSelectionState = {
-        ...state.categoricalSelectionState,
-        [action.metadataField]: {
-          ...state.categoricalSelectionState[action.metadataField],
-          categorySelected: newCategorySelected
-        }
-      };
-
-      // update the filter to match all selected options
-      const cat = newCategoricalSelectionState[action.metadataField];
-      state.dimensionMap[obsAnnoDimensionName(action.metadataField)].filterEnum(
-        selectedValuesForCategory(cat)
-      );
-
-      return {
-        ...state,
-        categoricalSelectionState: newCategoricalSelectionState
-      };
-    }
-    case "categorical metadata filter deselect": {
-      const newCategorySelected = Array.from(
-        state.categoricalSelectionState[action.metadataField].categorySelected
-      );
-      newCategorySelected[action.categoryIndex] = false;
-      const newCategoricalSelectionState = {
-        ...state.categoricalSelectionState,
-        [action.metadataField]: {
-          ...state.categoricalSelectionState[action.metadataField],
-          categorySelected: newCategorySelected
-        }
-      };
-
-      // update the filter to match all selected options
-      const cat = newCategoricalSelectionState[action.metadataField];
-      state.dimensionMap[obsAnnoDimensionName(action.metadataField)].filterEnum(
-        selectedValuesForCategory(cat)
-      );
-
-      return {
-        ...state,
-        categoricalSelectionState: newCategoricalSelectionState
-      };
-    }
-    case "categorical metadata filter none of these": {
-      const newCategoricalSelectionState = {
-        ...state.categoricalSelectionState,
-        [action.metadataField]: {
-          ...state.categoricalSelectionState[action.metadataField],
-          categorySelected: Array.from(
-            state.categoricalSelectionState[action.metadataField]
-              .categorySelected
-          ).fill(false)
-        }
-      };
-      state.dimensionMap[
-        obsAnnoDimensionName(action.metadataField)
-      ].filterNone();
-      return {
-        ...state,
-        categoricalSelectionState: newCategoricalSelectionState
-      };
-    }
-    case "categorical metadata filter all of these": {
-      const newCategoricalSelectionState = {
-        ...state.categoricalSelectionState,
-        [action.metadataField]: {
-          ...state.categoricalSelectionState[action.metadataField],
-          categorySelected: Array.from(
-            state.categoricalSelectionState[action.metadataField]
-              .categorySelected
-          ).fill(true)
-        }
-      };
-      state.dimensionMap[
-        obsAnnoDimensionName(action.metadataField)
-      ].filterAll();
-      return {
-        ...state,
-        categoricalSelectionState: newCategoricalSelectionState
-      };
-    }
-
-    /*******************************
-              Color Scale
-    *******************************/
-    case "color by categorical metadata":
-    case "color by continuous metadata": {
-      return {
-        ...state,
-        colorRGB: action.colors.rgb,
-        colorAccessor: action.colorAccessor,
-        colorScale: action.colorScale
-      };
-    }
-    case "color by expression": {
-      return {
-        ...state,
-        colorRGB: action.colors.rgb,
-        colorAccessor: action.gene,
-        colorScale: action.colorScale
-      };
-    }
 
     /*******************************
               Scatterplot

client/src/reducers/crossfilter.js

@@ -0,0 +1,189 @@
+import _ from "lodash";
+
+import Crossfilter from "../util/typedCrossfilter";
+import { World, ControlsHelpers } from "../util/stateManager";
+import {
+  layoutDimensionName,
+  obsAnnoDimensionName,
+  userDefinedDimensionName,
+  diffexpDimensionName,
+  makeContinuousDimensionName
+} from "../util/nameCreators";
+
+const CrossfilterReducer = (
+  state = null,
+  action,
+  nextSharedState,
+  prevSharedState
+) => {
+  switch (action.type) {
+    case "initial data load complete (universe exists)": {
+      const { world } = nextSharedState;
+      const crossfilter = World.createObsDimensions(
+        new Crossfilter(world.obsAnnotations),
+        world
+      );
+      return crossfilter;
+    }
+
+    case "reset World to eq Universe": {
+      const { userDefinedGenes, diffexpGenes } = prevSharedState.controls;
+      const { world } = nextSharedState;
+      const crossfilter = ControlsHelpers.createGeneDimensions(
+        userDefinedGenes,
+        diffexpGenes,
+        world,
+        prevSharedState.resetCache.crossfilter
+      );
+      return crossfilter;
+    }
+
+    case "set World to current selection": {
+      const { userDefinedGenes, diffexpGenes } = prevSharedState.controls;
+      const { world } = nextSharedState;
+      let crossfilter = new Crossfilter(world.obsAnnotations);
+      crossfilter = World.createObsDimensions(crossfilter, world);
+      crossfilter = ControlsHelpers.createGeneDimensions(
+        userDefinedGenes,
+        diffexpGenes,
+        world,
+        crossfilter
+      );
+      return crossfilter;
+    }
+
+    case "request user defined gene success": {
+      const { world } = prevSharedState;
+      const gene = action.data.genes[0];
+      return state.addDimension(
+        userDefinedDimensionName(gene),
+        "scalar",
+        world.varData.col(gene).asArray(),
+        Float32Array
+      );
+    }
+
+    case "request differential expression success": {
+      const { world } = prevSharedState;
+      const genes = _.map(action.data, d =>
+        world.varAnnotations.at(d[0], "name")
+      );
+      const crossfilter = _.reduce(
+        genes,
+        (xfltr, gene) =>
+          xfltr.addDimension(
+            diffexpDimensionName(gene),
+            "scalar",
+            world.varData.col(gene).asArray(),
+            Float32Array
+          ),
+        state
+      );
+      return crossfilter;
+    }
+
+    case "clear differential expression": {
+      const { world } = prevSharedState;
+      const crossfilter = _.reduce(
+        action.diffExp,
+        (xfltr, values) => {
+          const name = world.varAnnotations.at(values[0], "name");
+          return xfltr.delDimension(diffexpDimensionName(name));
+        },
+        state
+      );
+      return crossfilter;
+    }
+
+    case "clear user defined gene": {
+      return state.delDimension(userDefinedDimensionName(action.data));
+    }
+
+    case "clear all user defined genes": {
+      const { userDefinedGenes } = prevSharedState.controls;
+      const crossfilter = _.reduce(
+        userDefinedGenes,
+        (xfltr, gene) => xfltr.delDimension(userDefinedDimensionName(gene)),
+        state
+      );
+      return crossfilter;
+    }
+
+    case "graph brush selection change": {
+      const name = layoutDimensionName("XY");
+      const [x0, y0] = action.brushCoords.northwest;
+      const [x1, y1] = action.brushCoords.southeast;
+      return state.select(name, {
+        mode: "within-rect",
+        x0,
+        y0,
+        x1,
+        y1
+      });
+    }
+
+    case "lasso deselect":
+    case "graph brush deselect": {
+      const name = layoutDimensionName("XY");
+      return state.select(name, { mode: "all" });
+    }
+
+    case "lasso selection": {
+      const { polygon } = action;
+      const name = layoutDimensionName("XY");
+      if (polygon.length < 3) {
+        // single point or a line is not a polygon, and is therefore a deselect
+        return state.select(name, { mode: "all" });
+      }
+      return state.select(name, {
+        mode: "within-polygon",
+        polygon
+      });
+    }
+
+    case "continuous metadata histogram start":
+    case "continuous metadata histogram brush":
+    case "continuous metadata histogram end": {
+      const name = makeContinuousDimensionName(
+        action.continuousNamespace,
+        action.selection
+      );
+      // action.selection: metadata name being selected
+      // action.range: filter range, or null if deselected
+      if (!action.range) {
+        return state.select(name, { mode: "all" });
+      }
+      const [lo, hi] = action.range;
+      const newState = state.select(name, { mode: "range", lo, hi });
+      return newState;
+    }
+
+    case "categorical metadata filter select":
+    case "categorical metadata filter deselect": {
+      const { categoricalSelection } = nextSharedState;
+      const cat = categoricalSelection[action.metadataField];
+      return state.select(obsAnnoDimensionName(action.metadataField), {
+        mode: "exact",
+        values: ControlsHelpers.selectedValuesForCategory(cat)
+      });
+    }
+
+    case "categorical metadata filter none of these": {
+      return state.select(obsAnnoDimensionName(action.metadataField), {
+        mode: "none"
+      });
+    }
+
+    case "categorical metadata filter all of these": {
+      return state.select(obsAnnoDimensionName(action.metadataField), {
+        mode: "all"
+      });
+    }
+
+    default: {
+      return state;
+    }
+  }
+};
+
+export default CrossfilterReducer;

client/src/reducers/index.js

@@ -1,27 +1,89 @@
-// jshint esversion: 6
-import { combineReducers, createStore, applyMiddleware } from "redux";
+import { createStore, applyMiddleware } from "redux";
 import thunk from "redux-thunk";
-import { composeWithDevTools } from "redux-devtools-extension";
-import updateURLMiddleware from "../middleware/updateURLMiddleware";
-import updateCellColors from "../middleware/updateCellColors";
 
+import cascadeReducers from "./cascade";
+import undoable from "./undoable";
 import config from "./config";
+import universe from "./universe";
+import world from "./world";
+import categoricalSelection from "./categoricalSelection";
+import continuousSelection from "./continuousSelection";
+import crossfilter from "./crossfilter";
+import colors from "./colors";
 import differential from "./differential";
 import responsive from "./responsive";
 import controls from "./controls";
+import resetCache from "./resetCache";
 
-const Reducer = combineReducers({
-  config,
-  responsive,
-  controls,
-  differential
-});
+const ignoredActions = new Set([
+  // these actions will not affect history, ie, we will
+  // not snapshot history upon these actions.  These take
+  // precedent over `clearHistoryUponActions`
+  "url changed",
+  "interface reset started",
+  "initial data load start",
+  "configuration load complete",
+  "increment graph render counter",
+  "window resize",
 
-const store = createStore(
-  Reducer,
-  composeWithDevTools(
-    applyMiddleware(thunk, updateURLMiddleware, updateCellColors)
-  )
+  "lasso started",
+
+  "request differential expression success",
+
+  "expression load start",
+  "expression load success",
+  "expression load error",
+
+  "continuous metadata histogram brush",
+  "continuous metadata histogram end",
+
+  "request user defined gene started",
+  "request user defined gene success",
+  "request user defined gene error",
+  "bulk user defined gene complete",
+  "single user defined gene complete"
+]);
+
+const clearOnActions = new Set([
+  // history will be cleared when these actions occur
+  "initial data load complete (universe exists)",
+  "reset World to eq Universe",
+  "initial data load error"
+]);
+
+/* configuration for the undoable meta reducer */
+const undoableConfig = {
+  historyLimit: 50, // maximum history size
+  skipActionFilter: (state, action) => ignoredActions.has(action.type),
+  clearOnActionFilter: (state, action) => clearOnActions.has(action.type)
+};
+
+const Reducer = undoable(
+  cascadeReducers([
+    ["config", config],
+    ["universe", universe],
+    ["world", world],
+    ["categoricalSelection", categoricalSelection],
+    ["continuousSelection", continuousSelection],
+    ["crossfilter", crossfilter],
+    ["colors", colors],
+    ["controls", controls],
+    ["differential", differential],
+    ["responsive", responsive],
+    ["resetCache", resetCache]
+  ]),
+  [
+    "world",
+    "categoricalSelection",
+    "continuousSelection",
+    "crossfilter",
+    "colors",
+    "controls",
+    "differential"
+  ],
+  undoableConfig
 );
 
+const store = createStore(Reducer, applyMiddleware(thunk));
+
 export default store;

client/src/reducers/resetCache.js

@@ -0,0 +1,28 @@
+/*
+Reducer which caches derived state to be used in a reset
+*/
+
+const ResetCacheReducer = (
+  state = {
+    world: null,
+    crossfilter: null
+  },
+  action,
+  nextSharedState
+) => {
+  switch (action.type) {
+    case "initial data load complete (universe exists)": {
+      const { world, crossfilter } = nextSharedState;
+      return {
+        ...state,
+        world,
+        crossfilter
+      };
+    }
+    default: {
+      return state;
+    }
+  }
+};
+
+export default ResetCacheReducer;

client/src/reducers/undoable.js

@@ -0,0 +1,167 @@
+/*
+A redo/undo meta reducer for Redux.   Designed to work well with the cascadeReducer().
+
+Requires three parameters:
+* reducer - a reducer, which MUST return an object as state.
+* undoableKeys - an array of object keys (strings).   If any of these keys
+  are in the object/state returned by the reducer, they will be treated as
+  state to be made "undoable".
+* options - an optional object, which may contain the following parameters:
+    * historyLimit: max number of historical states to remember (aka max undo depth)
+    * skipActionFilter: filter function, (state, action) => bool.  If it returns
+      truthy, the current state will not be pushed onto the history stack.
+    * clearOnActionFilter: filter function, (state, action) => bool.  If it returns
+      truthy, the history state will be cleared as part of handling this action.
+
+skipActionFilter has precedence over clearOnActionFilter.
+
+This meta reducer accepts three actions types:
+* @@undoable/undo - move back in history
+* @@undoable/redo - move forward in history
+* @@undoable/clear - clear history
+*/
+
+const historyKeyPrefix = "@@undoable/";
+const pastKey = `${historyKeyPrefix}past`;
+const futureKey = `${historyKeyPrefix}future`;
+const defaultHistoryLimit = -100;
+
+const Undoable = (reducer, undoableKeys, options = {}) => {
+  let { historyLimit } = options;
+  if (!historyLimit) historyLimit = defaultHistoryLimit;
+  if (historyLimit > 0) historyLimit = -historyLimit;
+  const skipActionFilter = options.skipActionFilter || (() => false);
+  const clearOnActionFilter = options.clearOnActionFilter || (() => false);
+
+  if (!Array.isArray(undoableKeys) || undoableKeys.length === 0)
+    throw new Error("undoable keys array must be specified");
+  const undoableKeysSet = new Set(undoableKeys);
+
+  function undo(currentState) {
+    const past = currentState[pastKey];
+    const future = currentState[futureKey];
+    if (past.length === 0) return currentState;
+    const currentUndoableState = Object.entries(currentState).filter(kv =>
+      undoableKeysSet.has(kv[0])
+    );
+    const newPast = [...past];
+    const newState = newPast.pop();
+    const newFuture = push(future, currentUndoableState);
+    const nextState = {
+      ...currentState,
+      ...fromEntries(newState),
+      [pastKey]: newPast,
+      [futureKey]: newFuture
+    };
+    return nextState;
+  }
+
+  function redo(currentState) {
+    const past = currentState[pastKey] || [];
+    const future = currentState[futureKey] || [];
+    if (future.length === 0) return currentState;
+    const currentUndoableState = Object.entries(currentState).filter(kv =>
+      undoableKeysSet.has(kv[0])
+    );
+    const newFuture = [...future];
+    const newState = newFuture.pop();
+    const newPast = push(past, currentUndoableState);
+    const nextState = {
+      ...currentState,
+      ...fromEntries(newState),
+      [pastKey]: newPast,
+      [futureKey]: newFuture
+    };
+    return nextState;
+  }
+
+  function clear(currentState) {
+    return {
+      ...currentState,
+      [pastKey]: [],
+      [futureKey]: []
+    };
+  }
+
+  function skip(currentState, action) {
+    const past = currentState[pastKey] || [];
+    const res = reducer(currentState, action);
+    return {
+      ...res,
+      [pastKey]: past,
+      [futureKey]: []
+    };
+  }
+
+  function save(currentState, action) {
+    const past = currentState[pastKey] || [];
+    const currentUndoableState = Object.entries(currentState).filter(kv =>
+      undoableKeysSet.has(kv[0])
+    );
+    const res = reducer(currentState, action);
+    const newPast = push(past, currentUndoableState, historyLimit);
+    const nextState = {
+      ...res,
+      [pastKey]: newPast,
+      [futureKey]: []
+    };
+    return nextState;
+  }
+
+  return (
+    currentState = {
+      [pastKey]: [],
+      [futureKey]: []
+    },
+    action
+  ) => {
+    const aType = action.type;
+    switch (aType) {
+      case "@@undoable/undo": {
+        return undo(currentState, action);
+      }
+      case "@@undoable/redo": {
+        return redo(currentState, action);
+      }
+      case "@@undoable/clear": {
+        return clear(currentState, action);
+      }
+      default: {
+        if (skipActionFilter(currentState, action)) {
+          return skip(currentState, action);
+        }
+        if (clearOnActionFilter(currentState, action)) {
+          return clear(skip(currentState, action));
+        }
+        return save(currentState, action);
+      }
+    }
+  };
+};
+
+function push(arr, val, limit = undefined) {
+  /*
+  functional array push, with a max length limit to the new array.
+  Like Array.push, except it returns new array and discards as needed
+  to enforce the length limit.
+  */
+  const narr = arr.slice(limit);
+  narr.push(val);
+  return narr;
+}
+
+function fromEntries(arr) {
+  /*
+  Similar to Object.fromEntries, but only handles array.
+  This could be replaced with the standard fucnction once it
+  is widely available.   As of 3/20/2019, it has not yet
+  been released in the Chrome stable channel.
+  */
+  const obj = {};
+  for (let i = 0, l = arr.length; i < l; i += 1) {
+    obj[arr[i][0]] = arr[i][1];
+  }
+  return obj;
+}
+
+export default Undoable;

client/src/reducers/universe.js

@@ -0,0 +1,47 @@
+import _ from "lodash";
+
+import { ControlsHelpers } from "../util/stateManager";
+
+const Universe = (state = null, action, nextSharedState, prevSharedState) => {
+  switch (action.type) {
+    case "initial data load complete (universe exists)": {
+      const { universe } = action;
+      return universe;
+    }
+
+    case "expression load success": {
+      let { varData } = state;
+
+      // Load new expression data into the varData dataframes, if
+      // not already present.
+      _.forEach(action.expressionData, (val, key) => {
+        // If not already in universe.varData, save entire expression column
+        if (!varData.hasCol(key)) {
+          varData = varData.withCol(key, val);
+        }
+      });
+
+      // Prune size of varData "cache" if getting out of hand....
+      const { userDefinedGenes, diffexpGenes } = prevSharedState;
+      const allTheGenesWeNeed = _.uniq(
+        [].concat(
+          userDefinedGenes,
+          diffexpGenes,
+          Object.keys(action.expressionData)
+        )
+      );
+      varData = ControlsHelpers.pruneVarDataCache(varData, allTheGenesWeNeed);
+
+      return {
+        ...state,
+        varData
+      };
+    }
+
+    default: {
+      return state;
+    }
+  }
+};
+
+export default Universe;

client/src/reducers/world.js

@@ -0,0 +1,88 @@
+import _ from "lodash";
+
+import { World, ControlsHelpers } from "../util/stateManager";
+
+const WorldReducer = (
+  state = null,
+  action,
+  nextSharedState,
+  prevSharedState
+) => {
+  switch (action.type) {
+    case "initial data load complete (universe exists)": {
+      const { universe } = nextSharedState;
+      const world = World.createWorldFromEntireUniverse(universe);
+      return world;
+    }
+
+    case "reset World to eq Universe": {
+      return prevSharedState.resetCache.world;
+    }
+
+    case "set World to current selection": {
+      /* Set viewable world to be the currently selected data */
+      const world = World.createWorldFromCurrentSelection(
+        action.universe,
+        action.world,
+        action.crossfilter
+      );
+      return world;
+    }
+
+    case "expression load success": {
+      const { universe } = nextSharedState;
+      const universeVarData = universe.varData;
+      let worldVarData = state.varData;
+
+      // Load new expression data into the varData dataframes, if
+      // not already present.
+      _.forEach(action.expressionData, (val, key) => {
+        // If not already in world.varData, save sliced expression column
+        if (!worldVarData.hasCol(key)) {
+          // Slice if world !== universe, else just use whole column.
+          // Use the obsAnnotation index as the cut key, as we keep
+          // all world dataframes in sync.
+          let worldValSlice = val;
+          if (!World.worldEqUniverse(state, universe)) {
+            worldValSlice = universeVarData
+              .subset(state.obsAnnotations.rowIndex.keys(), [key], null)
+              .icol(0)
+              .asArray();
+          }
+
+          // Now build world's varData dataframe
+          worldVarData = worldVarData.withCol(
+            key,
+            worldValSlice,
+            state.obsAnnotations.rowIndex
+          );
+        }
+      });
+
+      // Prune size of varData "cache" if getting out of hand....
+      const { userDefinedGenes, diffexpGenes } = prevSharedState;
+      const allTheGenesWeNeed = _.uniq(
+        [].concat(
+          userDefinedGenes,
+          diffexpGenes,
+          Object.keys(action.expressionData)
+        )
+      );
+      worldVarData = ControlsHelpers.pruneVarDataCache(
+        worldVarData,
+        allTheGenesWeNeed
+      );
+
+      return {
+        ...state,
+        varData: worldVarData
+      };
+    }
+
+    default: {
+      return state;
+    }
+  }
+};
+
+export default WorldReducer;

client/src/util/dataframe/dataframe.js

@@ -0,0 +1,574 @@
+import { IdentityInt32Index, isLabelIndex } from "./labelIndex";
+// weird cross-dependency that we should clean up someday...
+import { sort } from "../typedCrossfilter/sort";
+import { isTypedArray, isArrayOrTypedArray, callOnceLazy } from "./util";
+import { summarizeContinuous, summarizeCategorical } from "./summarize";
+
+/*
+Dataframe is an immutable 2D matrix similiar to Python Pandas Dataframe,
+but (currently) without all of the surrounding support functions.
+Data is stored in column-major layout, and each column is monomorphic.
+
+It supports:
+* Relatively efficient creation, cloning and subsetting
+* Very efficient columnar access (eg, sum down a column), and access
+  to the underlying column arrays.
+* Data access by row/col offset or label.  Labels are reasonably well
+  optimized for both numeric lables and arbitrary (eg, sting) labels.
+
+It does not currently support:
+* Views on matrix subset - for currently known access patterns,
+  it is more effiicent to copy on subsetting, optimizing for access
+  speed over memory use.
+* JS iterators - they are too slow.  Use explicit iteration over
+  offest or labels.
+
+Important assumptions embedded in the API:
+* Columns are implicitly categorical if they are a JS Array and numeric
+  (aka continuous) if they are a TypedArray.
+
+There are three index types for row/col indexing:
+* IdentityInt32Index - noop index, where the index label is the offset.
+* KeyIndex - index arbitrary JS objects.
+* DenseInt32Index - integer indexing.  Optimization over KeyIndex as it uses
+  Int32Array as a back-map to offsets.  This means that the index array
+  must be sized to [minLabel, maxLabel), so this is only useful when the label
+  range is relatively close the underlying offset range [minOffset, maxOffset).
+
+All private functions/methods/fields are prefixed by '__', eg, __compile().
+Don't use them outside of this file.
+
+Simple example:
+
+  // default indexing is integer offset.
+  const df = Dataframe.create([2,2], [['a', 'b'], [0, 1]])
+  console.log(df.at(0,0));  // outputs: a
+  console.log(df.col(1).asArray());   // outputs: [0, 1]
+
+  // KeyIndex
+  const df = new Dataframe([1,2], [['a'], ['b']], null, new KeyIndex(['A', 'B']))
+  console.log(df.at(0, 'A')); // outputs: a
+  console.log(df.col('A').asArray(); // outputs: ['a']
+
+Performance tuning is primarily focused on columnar access patterns, which is the
+dominant pattern in cellxgene.
+*/
+
+/**
+Dataframe
+**/
+
+class Dataframe {
+  /**
+  Constructors & factories
+  **/
+
+  constructor(dims, columnarData, rowIndex = null, colIndex = null) {
+    /*
+    The base constructor is relatively hard to use - as an alternative,
+    see factory methods and clone/slice, below.
+
+    Parameters:
+      * dims - 2D array describing intendend dimensionality: [nRows,nCols].
+      * columnarData - JS array, nCols in length, containing array
+        or TypedArray of length nRows.
+      * rowIndex/colIndex - null (create default index using offsets as key),
+        or a caller-provided index.
+    All columns and indices must have appropriate dimensionality.
+    */
+    const [nRows, nCols] = dims;
+    if (nRows < 0 || nCols < 0) {
+      throw new RangeError("Dataframe dimensions must be positive");
+    }
+    if (!rowIndex) {
+      rowIndex = new IdentityInt32Index(nRows);
+    }
+    if (!colIndex) {
+      colIndex = new IdentityInt32Index(nCols);
+    }
+    Dataframe.__errorChecks(dims, columnarData, rowIndex, colIndex);
+
+    this.__columns = Array.from(columnarData);
+    this.dims = dims;
+    this.length = nRows; // convenience accessor for row dimension
+    this.rowIndex = rowIndex;
+    this.colIndex = colIndex;
+
+    this.__compile();
+  }
+
+  static __errorChecks(dims, columnarData, rowIndex, colIndex) {
+    const [nRows, nCols] = dims;
+
+    /* check for expected types */
+    if (!Array.isArray(columnarData)) {
+      throw new TypeError("Dataframe constructor requires array of columns");
+    }
+    if (!columnarData.every(c => isArrayOrTypedArray(c))) {
+      throw new TypeError("Dataframe columns must all be Array or TypedArray");
+    }
+    if (!isLabelIndex(rowIndex)) {
+      throw new TypeError("Dataframe rowIndex is an unsupported type.");
+    }
+    if (!isLabelIndex(colIndex)) {
+      throw new TypeError("Dataframe colIndex is an unsupported type.");
+    }
+
+    /* check for expected dimensionality / size */
+    if (
+      nCols !== columnarData.length ||
+      !columnarData.every(c => c.length === nRows)
+    ) {
+      throw new RangeError(
+        "Dataframe dimension does not match provided data shape"
+      );
+    }
+    if (nRows !== rowIndex.size()) {
+      throw new RangeError(
+        "Dataframe rowIndex must have same size as underlying data"
+      );
+    }
+    if (nCols !== colIndex.size()) {
+      throw new RangeError(
+        "Dataframe colIndex must have same size as underlying data"
+      );
+    }
+  }
+
+  __compile() {
+    /*
+    Compile data accessors for each column.
+
+    Each column accessor is a function which will lookup data by
+    index (ie, is equivalent to dataframe.get(row, col), where 'col'
+    is fixed.
+
+    In addition, each column accessor has several functions:
+
+    asArray() -- return the entire column as a native Array or TypedArray.
+      Crucially, this native array only supports label indexing.
+      Example:
+        const arr = df.col('a').asArray();
+
+    has(rlabel) -- return boolean indicating of the row label
+      is contained within the column.  Example:
+        const isInColumn = df.col('a').includes(99)
+      For the default offset indexing, this is identical to:
+        const isInColumn = (99 > 0) && (99 < df.nRows);
+
+    ihas(roffset) -- same as has(), but accepts a row offset
+      instead of a row label.
+
+    indexOf(value) -- return the label (not offset) of the first instance of
+      'value' in the column.  If you want the offset, just use the builtin JS
+      indexOf() function, available on both Array and TypedArray.
+
+    iget(offset) -- return the value at 'offset'
+
+    */
+    const { getOffset, getLabel } = this.rowIndex;
+    this.__columnsAccessor = this.__columns.map(column => {
+      const { length } = column;
+
+      /* get value by row label */
+      const get = function get(rlabel) {
+        return column[getOffset(rlabel)];
+      };
+
+      /* get value by row offset */
+      const iget = function iget(roffset) {
+        return column[roffset];
+      };
+
+      /* full column array access */
+      const asArray = function asArray() {
+        return column;
+      };
+
+      /* test for row label inclusion in column */
+      const has = function has(rlabel) {
+        const offset = getOffset(rlabel);
+        return offset >= 0 && offset < length;
+      };
+
+      const ihas = function ihas(offset) {
+        return offset >= 0 && offset < length;
+      };
+
+      /*
+      return first label (index) at which the value is found in this column,
+      or undefined if not found.
+
+      NOTE: not found return is DIFFERENT than the default Array.indexOf as
+      -1 is a plausible Dataframe row/col label.
+      */
+      const indexOf = function indexOf(value) {
+        const offset = column.indexOf(value);
+        if (offset === -1) {
+          return undefined;
+        }
+        return getLabel(offset);
+      };
+
+      /*
+      Summarize the column data. Lazy eval;
+      */
+      const summarize = callOnceLazy(() =>
+        isTypedArray(column)
+          ? summarizeContinuous(column)
+          : summarizeCategorical(column)
+      );
+
+      get.summarize = summarize;
+      get.asArray = asArray;
+      get.has = has;
+      get.ihas = ihas;
+      get.indexOf = indexOf;
+      get.iget = iget;
+      return get;
+    });
+  }
+
+  clone() {
+    /*
+    Clone this dataframe
+    */
+    return new this.constructor(
+      this.dims,
+      [...this.__columns],
+      this.rowIndex,
+      this.colIndex
+    );
+  }
+
+  withCol(label, colData, withRowIndex = null) {
+    /*
+    Create a new DF, which is `this` plus the new column. Example:
+    const newDf = df.withCol("foo", [1,2,3]);
+
+    Dimensionality of new column must match existing dataframe.
+
+    Special case: empty dataframe will accept any size column.  Example:
+    const newDf = Dataframe.empty().withCol("foo", [1,2,3]);
+
+    If `withRowIndex` specified, the provided index will become the
+    rowIndex for the newly created dataframe.   If not specified,
+    the rowIndex from `this` will be used (ie, the rowIndex is
+    unchanged).
+    */
+    let dims;
+    let rowIndex;
+    if (this.isEmpty()) {
+      dims = [colData.length, 1];
+      rowIndex = null;
+    } else {
+      dims = [this.dims[0], this.dims[1] + 1];
+      ({ rowIndex } = this);
+    }
+
+    if (withRowIndex) {
+      rowIndex = withRowIndex;
+    }
+
+    const columns = [...this.__columns];
+    columns.push(colData);
+    const colIndex = this.colIndex.withLabel(label);
+    return new this.constructor(dims, columns, rowIndex, colIndex);
+  }
+
+  dropCol(label) {
+    /*
+    Create a new dataframe, omitting one columns.
+
+    const newDf = df.dropCol("colors");
+    */
+    const dims = [this.dims[0], this.dims[1] - 1];
+    const coffset = this.colIndex.getOffset(label);
+    const columns = [...this.__columns];
+    columns.splice(coffset, 1);
+    const colIndex = this.colIndex.dropLabel(label);
+    return new this.constructor(dims, columns, this.rowIndex, colIndex);
+  }
+
+  static empty(rowIndex = null, colIndex = null) {
+    return new Dataframe([0, 0], [], rowIndex, colIndex);
+  }
+
+  static create(dims, columnarData) {
+    /*
+    Create a dataframe from raw columnar data.  All column arrays
+    must have the same length.   Identity indexing will be used.
+
+    Example:
+    const df = Dataframe.create([2,2], [new Uint32Array(2), new Float32Array(2)]);
+    */
+    return new Dataframe(dims, columnarData, null, null);
+  }
+
+  __subset(rowOffsets, colOffsets, withRowIndex) {
+    const dims = [...this.dims];
+
+    const getSortedLabelAndOffsets = (offsets, index) => {
+      /*
+      Given offsets, return both offsets and associated lables,
+      sorted by offset.
+      */
+      if (!offsets) {
+        return [null, null];
+      }
+      const sortedOffsets = sort(offsets);
+      const sortedLabels = new Array(sortedOffsets.length);
+      for (let i = 0, l = sortedOffsets.length; i < l; i += 1) {
+        sortedLabels[i] = index.getLabel(sortedOffsets[i]);
+      }
+      return [sortedLabels, sortedOffsets];
+    };
+
+    let { colIndex } = this;
+    if (colOffsets) {
+      let colLabels;
+      [colLabels, colOffsets] = getSortedLabelAndOffsets(
+        colOffsets,
+        this.colIndex
+      );
+      dims[1] = colOffsets.length;
+      colIndex = this.colIndex.subsetLabels(colLabels);
+    }
+
+    let { rowIndex } = this;
+    if (withRowIndex) rowIndex = withRowIndex;
+    if (rowOffsets) {
+      let rowLabels;
+      [rowLabels, rowOffsets] = getSortedLabelAndOffsets(
+        rowOffsets,
+        this.rowIndex
+      );
+      dims[0] = rowLabels.length;
+      if (!withRowIndex) rowIndex = this.rowIndex.subsetLabels(rowLabels);
+    }
+
+    /* subset columns */
+    let columns = this.__columns;
+    if (colOffsets) {
+      columns = new Array(colOffsets.length);
+      for (let i = 0, l = colOffsets.length; i < l; i += 1) {
+        columns[i] = this.__columns[colOffsets[i]];
+      }
+    }
+
+    /* subset rows */
+    if (rowOffsets) {
+      columns = columns.map(col => {
+        const newCol = new col.constructor(rowOffsets.length);
+        for (let i = 0, l = rowOffsets.length; i < l; i += 1) {
+          newCol[i] = col[rowOffsets[i]];
+        }
+        return newCol;
+      });
+    }
+    return new Dataframe(dims, columns, rowIndex, colIndex);
+  }
+
+  subset(rowLabels, colLabels = null, withRowIndex = null) {
+    /*
+    Subset by row/col labels.
+
+    withRowIndex allows assignment of new row index during subset operation.
+    If withRowIndex === null, it will reset the index to identity (offset)
+    indexing.  if withRowIndex is a label index object, it will be used
+    for the new dataframe.
+    */
+    const toOffsets = (labels, index) => {
+      if (!labels) {
+        return null;
+      }
+      return labels.map(label => {
+        const off = index.getOffset(label);
+        if (off === undefined) {
+          throw new RangeError(`unknown label: ${label}`);
+        }
+        return off;
+      });
+    };
+
+    const rowOffsets = toOffsets(rowLabels, this.rowIndex);
+    const colOffsets = toOffsets(colLabels, this.colIndex);
+    return this.__subset(rowOffsets, colOffsets, withRowIndex);
+  }
+
+  isubset(rowOffsets, colOffsets = null, withRowIndex = null) {
+    /*
+    Subset by row/col offset.
+
+    withRowIndex allows assignment of new row index during subset operation.
+    If withRowIndex === null, it will reset the index to identity (offset)
+    indexing.  if withRowIndex is a label index object, it will be used
+    for the new dataframe.
+    */
+    return this.__subset(rowOffsets, colOffsets, withRowIndex);
+  }
+
+  isubsetMask(rowMask, colMask = null, withRowIndex = null) {
+    /*
+    Subset on row/column based upon a truthy/falsey array (a mask).
+
+    withRowIndex allows assignment of new row index during subset operation.
+    If withRowIndex === null, it will reset the index to identity (offset)
+    indexing.  if withRowIndex is a label index object, it will be used
+    for the new dataframe.
+    */
+    const [nRows, nCols] = this.dims;
+    if (
+      (rowMask && rowMask.length !== nRows) ||
+      (colMask && colMask.length !== nCols)
+    ) {
+      throw new RangeError("boolean arrays must match row/col dimensions");
+    }
+
+    /* convert masks to lists - method wastes space, but is fast */
+    const toList = (mask, maxSize) => {
+      if (!mask) {
+        return null;
+      }
+      const list = new Int32Array(maxSize);
+      let elems = 0;
+      for (let i = 0, l = mask.length; i < l; i += 1) {
+        if (mask[i]) {
+          list[elems] = i;
+          elems += 1;
+        }
+      }
+      return new Int32Array(list.buffer, 0, elems);
+    };
+    const rowOffsets = toList(rowMask, nRows);
+    const colOffsets = toList(colMask, nCols);
+    return this.__subset(rowOffsets, colOffsets, withRowIndex);
+  }
+
+  /**
+  Data access with row/col.
+  **/
+
+  col(columnLabel) {
+    /*
+    Return accessor bound to a column.  Allows random row access
+    based upon the row indexing.  Returns undefined if the
+    columnLabel is not present in the dataframe.
+
+    Example for a dataframe with string labeled columns, and
+    default (offset) indices for rows (eg, [0, 'foo'])
+
+      const getValue = df.col('foo');
+      for (let r = 0; r < df.nRows; r += 1) {
+        console.log(r, getValue(r));
+      }
+
+    See __compile() for the functions available in a column accessor.
+    */
+    const coff = this.colIndex.getOffset(columnLabel);
+    return this.__columnsAccessor[coff];
+  }
+
+  icol(columnOffset) {
+    /*
+    Return column accessor by offset.
+    */
+    return this.__columnsAccessor[columnOffset];
+  }
+
+  at(r, c) {
+    /*
+    Access a single value, for a row/col label pair.
+
+    For performance reasons, there are no bounds or existance
+    checks on labels, and no defined behavior when these are supplied.
+    May return undefined, throw an Error, or do something else for
+    non-existant labels.  If you want predictable out-of-bounds
+    behavior, use has(), eg,
+
+    const myVal = df.has(r,l) ? df.at(r,l) : undefined;
+    */
+    const coff = this.colIndex.getOffset(c);
+    const roff = this.rowIndex.getOffset(r);
+    return this.__columns[coff][roff];
+  }
+
+  iat(r, c) {
+    /*
+    Access a single value, for a row/col offset (integer) position.
+
+    For performance reasons, there are no bounds checks on row/col offsets
+    or other well-defined behavior for out-of-bounds values.  If you want
+    well-defined bounds checking, use ihas(), eg,
+
+    const myVal = df.ihas(r, c) ? df.iat(r, c) : undefined;
+    */
+    return this.__columns[c][r];
+  }
+
+  has(r, c) {
+    /*
+    Test if row/col labels exist in the dataframe - returns true/false
+    */
+    const [nRows, nCols] = this.dims;
+    const coff = this.colIndex.getOffset(c);
+    const roff = this.rowIndex.getOffset(r);
+    return coff >= 0 && coff < nCols && roff >= 0 && roff < nRows;
+  }
+
+  ihas(r, c) {
+    /*
+    Test if row/col offset (integer) position exists in the
+    dataframe - returns true/false
+    */
+    const [nRows, nCols] = this.dims;
+    return c >= 0 && c < nCols && r >= 0 && r < nRows;
+  }
+
+  hasCol(c) {
+    /*
+    Test if col label exists - return true/false
+    */
+    return !!this.col(c);
+  }
+
+  isEmpty() {
+    /*
+    Return true if this is an empty dataframe, ie, has dimensions [0,0]
+    */
+    const [rows, cols] = this.dims;
+    return rows === 0 && cols === 0;
+  }
+
+  /****
+  Functional (map/reduce/etc) data access
+
+  XXX: not yet implemented, as there is no clear use case.   Can easily
+  add these as useful.
+  ****/
+
+  /*
+  Map & reduce of column or row
+
+  XXX TODO remainder of map/reduce functions:  mapCol, mapRow, reduceRow, ...
+  */
+  /* comment out until we have a use for this
+
+  reduceCol(clabel, callback, initialValue) {
+    const coff = this.colIndex.getOffset(clabel);
+    const column = this.__columns[coff];
+    let start = 0;
+    let acc = initialValue;
+    if (initialValue === undefined) {
+      acc = column[0];
+      start = 1;
+    }
+    for (let i = start, l = column.length; i < l; i += 1) {
+      acc = callback(acc, column[i]);
+    }
+    return acc;
+  }
+  */
+}
+
+export default Dataframe;

client/src/util/dataframe/index.js

@@ -0,0 +1,2 @@
+export { default as Dataframe } from "./dataframe";
+export { DenseInt32Index, IdentityInt32Index, KeyIndex } from "./labelIndex";

client/src/util/dataframe/labelIndex.js

@@ -0,0 +1,244 @@
+/**
+Label indexing - map a label to & from an integer offset.  See Dataframe
+for how this is used.
+**/
+
+/*
+Private utility functions
+*/
+function extent(tarr) {
+  let min = 0x7fffffff;
+  let max = ~min; // eslint-disable-line no-bitwise
+  for (let i = 0, l = tarr.length; i < l; i += 1) {
+    const v = tarr[i];
+    if (v < min) {
+      min = v;
+    }
+    if (v > max) {
+      max = v;
+    }
+  }
+  return [min, max];
+}
+
+function fillRange(arr, start = 0) {
+  const larr = arr;
+  for (let i = 0, l = larr.length; i < l; i += 1) {
+    larr[i] = i + start;
+  }
+  return larr;
+}
+
+/* eslint-disable class-methods-use-this */
+class IdentityInt32Index {
+  /*
+  identity/noop index, with small assumptions that labels are int32
+  */
+  constructor(maxOffset) {
+    this.maxOffset = maxOffset;
+  }
+
+  keys() {
+    // memoize
+    const k = fillRange(new Int32Array(this.maxOffset));
+    this.keys = function keys() {
+      return k;
+    };
+    return k;
+  }
+
+  getOffset(i) {
+    // label to offset
+    return i;
+  }
+
+  getLabel(i) {
+    // offset to label
+    return i;
+  }
+
+  size() {
+    return this.maxOffset;
+  }
+
+  __promote(labelArray) {
+    /*
+    time/space decision - based on the resulting density
+    */
+    const [minLabel, maxLabel] = extent(labelArray);
+    const labelSpaceSize = maxLabel - minLabel + 1;
+    const density = labelSpaceSize / this.maxOffset;
+    /* 0.1 is a magic number, that needs testing to optimize */
+    if (density < 0.1) {
+      return new KeyIndex(labelArray);
+    }
+    return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
+  }
+
+  subsetLabels(labelArray) {
+    return this.__promote(labelArray);
+  }
+
+  withLabel(label) {
+    if (label === this.maxOffset) {
+      return new IdentityInt32Index(label + 1);
+    }
+    return this.__promote([...this.keys(), label]);
+  }
+
+  dropLabel(label) {
+    if (label === this.maxOffset - 1) {
+      return new IdentityInt32Index(label);
+    }
+    const labelArray = [...this.keys()];
+    labelArray.splice(labelArray.indexOf(label), 1);
+    return this.__promote(labelArray);
+  }
+}
+/* eslint-enable class-methods-use-this */
+
+/* eslint-disable class-methods-use-this */
+class DenseInt32Index {
+  /*
+  DenseInt32Index indexes integer labels, and uses Int32Array typed arrays
+  for both forward and reverse indexing.   This means that the min/max range
+  of the forward index labels must be known a priori (so that the index
+  array can be pre-allocated).
+  */
+  constructor(labels, labelRange = null) {
+    if (labels.constructor !== Int32Array) {
+      labels = new Int32Array(labels);
+    }
+
+    if (!labelRange) {
+      labelRange = extent(labels);
+    }
+    const [minLabel, maxLabel] = labelRange;
+    const labelSpaceSize = maxLabel - minLabel + 1;
+    const index = new Int32Array(labelSpaceSize).fill(-1);
+    for (let i = 0, l = labels.length; i < l; i += 1) {
+      const label = labels[i];
+      index[label - minLabel] = i;
+    }
+
+    this.minLabel = minLabel;
+    this.rindex = labels;
+    this.index = index;
+    this.__compile();
+  }
+
+  __compile() {
+    const { minLabel, index, rindex } = this;
+    this.getOffset = function getOffset(l) {
+      return index[l - minLabel];
+    };
+    this.getLabel = function getLabel(i) {
+      return rindex[i];
+    };
+  }
+
+  keys() {
+    return this.rindex;
+  }
+
+  size() {
+    return this.rindex.length;
+  }
+
+  __promote(labelArray) {
+    /*
+    time/space decision - if we are going to use less than 10% of the
+    dense index space, switch to a KeyIndex (which is slower, but uses
+    less memory for sparse label spaces).
+    */
+    const [minLabel, maxLabel] = extent(labelArray);
+    const labelSpaceSize = maxLabel - minLabel + 1;
+    const density = labelSpaceSize / this.rindex.length;
+    /* 0.1 is a magic number, that needs testing to optimize */
+    if (density < 0.1) {
+      return new KeyIndex(labelArray);
+    }
+    return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
+  }
+
+  subsetLabels(labelArray) {
+    return this.__promote(labelArray);
+  }
+
+  withLabel(label) {
+    return this.__promote([...this.keys(), label]);
+  }
+
+  dropLabel(label) {
+    const labelArray = [...this.keys()];
+    labelArray.splice(labelArray.indexOf(label), 1);
+    return this.__promote(labelArray);
+  }
+}
+/* eslint-enable class-methods-use-this */
+
+/* eslint-disable class-methods-use-this */
+class KeyIndex {
+  /*
+  KeyIndex indexes arbitrary JS primitive types, and uses a Map()
+  as its core data structure.
+  */
+  constructor(labels) {
+    const index = new Map();
+    if (labels === undefined) {
+      labels = [];
+    }
+    const rindex = labels;
+    labels.forEach((v, i) => {
+      index.set(v, i);
+    });
+
+    this.index = index;
+    this.rindex = rindex;
+    this.__compile();
+  }
+
+  __compile() {
+    const { index, rindex } = this;
+    this.getOffset = function getOffset(k) {
+      return index.get(k);
+    };
+    this.getLabel = function getLabel(i) {
+      return rindex[i];
+    };
+  }
+
+  keys() {
+    return this.rindex;
+  }
+
+  size() {
+    return this.rindex.length;
+  }
+
+  subsetLabels(labelArray) {
+    return new KeyIndex(labelArray);
+  }
+
+  withLabel(label) {
+    return new KeyIndex([...this.rindex, label]);
+  }
+
+  dropLabel(label) {
+    const idx = this.rindex.indexOf(label);
+    const labelArray = [...this.rindex];
+    labelArray.splice(idx, 1);
+    return new KeyIndex(labelArray);
+  }
+}
+/* eslint-enable class-methods-use-this */
+
+function isLabelIndex(i) {
+  return (
+    i instanceof IdentityInt32Index ||
+    i instanceof DenseInt32Index ||
+    i instanceof KeyIndex
+  );
+}
+
+export { DenseInt32Index, IdentityInt32Index, KeyIndex, isLabelIndex };

client/src/util/dataframe/summarize.js

@@ -0,0 +1,57 @@
+/*
+Private dataframe support functions
+*/
+
+export function summarizeContinuous(col) {
+  let min;
+  let max;
+  let nan = 0;
+  let pinf = 0;
+  let ninf = 0;
+  if (col) {
+    for (let r = 0, l = col.length; r < l; r += 1) {
+      const val = Number(col[r]);
+      if (Number.isFinite(val)) {
+        if (min === undefined) {
+          min = val;
+          max = val;
+        } else {
+          min = val < min ? val : min;
+          max = val > max ? val : max;
+        }
+      } else if (Number.isNaN(val)) {
+        nan += 1;
+      } else if (val > 0) {
+        pinf += 1;
+      } else {
+        ninf += 1;
+      }
+    }
+  }
+  return {
+    categorical: false,
+    min,
+    max,
+    nan,
+    pinf,
+    ninf
+  };
+}
+
+export function summarizeCategorical(col) {
+  const categoryCounts = new Map();
+  if (col) {
+    for (let r = 0, l = col.length; r < l; r += 1) {
+      const val = col[r];
+      let curCount = categoryCounts.get(val);
+      if (curCount === undefined) curCount = 0;
+      categoryCounts.set(val, curCount + 1);
+    }
+  }
+  return {
+    categorical: true,
+    categories: [...categoryCounts.keys()],
+    categoryCounts,
+    numCategories: categoryCounts.size
+  };
+}

client/src/util/dataframe/util.js

@@ -0,0 +1,28 @@
+/*
+Private utility code for dataframe
+*/
+
+export function isTypedArray(x) {
+  return (
+    ArrayBuffer.isView(x) &&
+    Object.prototype.toString.call(x) !== "[object DataView]"
+  );
+}
+
+export function isArrayOrTypedArray(x) {
+  return Array.isArray(x) || isTypedArray(x);
+}
+
+export function callOnceLazy(f) {
+  let value;
+  let calledOnce = false;
+  const result = function result(...args) {
+    if (!calledOnce) {
+      value = f(...args);
+      calledOnce = true;
+    }
+    return value;
+  };
+
+  return result;
+}

client/src/util/stateManager/colorHelpers.js

@@ -0,0 +1,122 @@
+/*
+Helper functions for the embedded graph colors
+*/
+import _ from "lodash";
+import * as d3 from "d3";
+import { interpolateRainbow, interpolateCool } from "d3-scale-chromatic";
+import * as globals from "../../globals";
+import parseRGB from "../parseRGB";
+import finiteExtent from "../finiteExtent";
+
+/*
+create new colors state object.   Paramters:
+  - world - current world object
+  - mode - color-by mode.  One of: null, "color by expression",
+    "color by continuous metadata", "color by categorical metadata"
+  -
+*/
+function createColors(world, colorMode = null, colorAccessor = null) {
+  switch (colorMode) {
+    case "color by categorical metadata": {
+      return createColorsByCategoricalMetadata(world, colorAccessor);
+    }
+    case "color by continuous metadata": {
+      return createColorsByContinuousMetadata(world, colorAccessor);
+    }
+    case "color by expression": {
+      return createColorsByExpression(world, colorAccessor);
+    }
+    default: {
+      const defaultCellColor = parseRGB(globals.defaultCellColor);
+      return {
+        rgb: new Array(world.nObs).fill(defaultCellColor),
+        scale: undefined
+      };
+    }
+  }
+}
+
+function createColorsByCategoricalMetadata(world, accessor) {
+  const { categories } = _.filter(world.schema.annotations.obs, {
+    name: accessor
+  })[0];
+
+  const scale = d3
+    .scaleSequential(interpolateRainbow)
+    .domain([0, categories.length]);
+
+  /* pre-create colors - much faster than doing it for each obs */
+  const colors = categories.reduce((acc, cat, idx) => {
+    acc[cat] = parseRGB(scale(idx));
+    return acc;
+  }, {});
+
+  const rgb = new Array(world.nObs);
+  const data = world.obsAnnotations.col(accessor).asArray();
+  for (let i = 0, len = world.obsAnnotations.length; i < len; i += 1) {
+    const cat = data[i];
+    rgb[i] = colors[cat];
+  }
+  return { rgb, scale };
+}
+
+function createColorsByContinuousMetadata(world, accessor) {
+  const colorBins = 100;
+  const col = world.obsAnnotations.col(accessor);
+  const { min, max } = col.summarize();
+  const scale = d3
+    .scaleQuantile()
+    .domain([min, max])
+    .range(_.range(colorBins - 1, -1, -1));
+
+  /* pre-create colors - much faster than doing it for each obs */
+  const colors = new Array(colorBins);
+  for (let i = 0; i < colorBins; i += 1) {
+    colors[i] = parseRGB(interpolateCool(i / colorBins));
+  }
+
+  const nonFiniteColor = parseRGB(globals.nonFiniteCellColor);
+  const rgb = new Array(world.nObs);
+  const data = col.asArray();
+  for (let i = 0, len = world.obsAnnotations.length; i < len; i += 1) {
+    const val = data[i];
+    if (Number.isFinite(val)) {
+      const c = scale(val);
+      rgb[i] = colors[c];
+    } else {
+      rgb[i] = nonFiniteColor;
+    }
+  }
+  return { rgb, scale };
+}
+
+function createColorsByExpression(world, accessor) {
+  const expression = world.varData.col(accessor).asArray();
+  const colorBins = 100;
+  const [min, max] = finiteExtent(expression);
+  const scale = d3
+    .scaleQuantile()
+    .domain([min, max])
+    .range(_.range(colorBins - 1, -1, -1));
+
+  /* pre-create colors - much faster than doing it for each obs */
+  const colors = new Array(colorBins);
+  for (let i = 0; i < colorBins; i += 1) {
+    colors[i] = parseRGB(interpolateCool(i / colorBins));
+  }
+  const nonFiniteColor = parseRGB(globals.nonFiniteCellColor);
+
+  const rgb = new Array(world.nObs);
+  for (let i = 0, len = expression.length; i < len; i += 1) {
+    const e = expression[i];
+    if (Number.isFinite(e)) {
+      const c = scale(e);
+      rgb[i] = colors[c];
+    } else {
+      rgb[i] = nonFiniteColor;
+    }
+  }
+  return { rgb, scale };
+}
+
+export default createColors;

client/src/util/stateManager/controlsHelpers.js

@@ -0,0 +1,169 @@
+/*
+Helper functions for the controls reducer
+*/
+
+import _ from "lodash";
+
+import * as globals from "../../globals";
+import { fillRange } from "../typedCrossfilter/util";
+import {
+  userDefinedDimensionName,
+  diffexpDimensionName
+} from "../nameCreators";
+
+/*
+Selection state for categoricals are tracked in an Object that
+has two main components for each category:
+1. mapping of option value to an index
+2. array of bool selection state by index
+Remember that option values can be ANY js type, except undefined/null.
+
+  {
+    _category_name_1: {
+      // map of option value to index
+      categoryIndices: Map([
+        catval1: index,
+        ...
+      ])
+
+      // index->selection true/false state
+      categorySelected: [ true/false, true/false, ... ]
+
+      // number of options
+      numCategories: number,
+
+      // isTruncated - true if the options for selection has
+      // been truncated (ie, was too large to implement)
+    }
+  }
+*/
+function topNCategories(summary) {
+  const counts = _.map(summary.categories, cat =>
+    summary.categoryCounts.get(cat)
+  );
+  const sortIndex = fillRange(new Array(summary.numCategories)).sort(
+    (a, b) => counts[b] - counts[a]
+  );
+  const sortedCategories = _.map(sortIndex, i => summary.categories[i]);
+  const sortedCounts = _.map(sortIndex, i => counts[i]);
+  const N = globals.maxCategoricalOptionsToDisplay;
+
+  if (sortedCategories.length < N) {
+    return [sortedCategories, sortedCounts];
+  }
+  return [sortedCategories.slice(0, N), sortedCounts.slice(0, N)];
+}
+
+export function createCategoricalSelection(maxCategoryItems, world) {
+  const res = {};
+  _.forEach(world.obsAnnotations.colIndex.keys(), key => {
+    const summary = world.obsAnnotations.col(key).summarize();
+    if (summary.categories) {
+      const isColorField = key.includes("color") || key.includes("Color");
+      const isSelectableCategory =
+        !isColorField &&
+        key !== "name" &&
+        summary.categories.length < maxCategoryItems;
+      if (isSelectableCategory) {
+        const [categoryValues, categoryCounts] = topNCategories(summary);
+        const categoryIndices = new Map(categoryValues.map((v, i) => [v, i]));
+        const numCategories = categoryIndices.size;
+        const categorySelected = new Array(numCategories).fill(true);
+        const isTruncated = categoryValues.length < summary.numCategories;
+        res[key] = {
+          categoryValues, // array: of natively typed category values
+          categoryIndices, // map: category value (native type) -> category index
+          categorySelected, // array: t/f selection state
+          numCategories, // number: of categories
+          isTruncated, // bool: true if list was truncated
+          categoryCounts // array: cardinality of each category
+        };
+      }
+    }
+  });
+  return res;
+}
+
+/*
+given a categoricalSelection, return the list of all category values
+where selection state is true (ie, they are selected).
+*/
+export function selectedValuesForCategory(categorySelectionState) {
+  const selectedValues = _([...categorySelectionState.categoryIndices])
+    .filter(tuple => categorySelectionState.categorySelected[tuple[1]])
+    .map(tuple => tuple[0])
+    .value();
+  return selectedValues;
+}
+
+/*
+build a crossfilter dimensions for all gene expression related dimensions.
+*/
+export function createGeneDimensions(
+  userDefinedGenes,
+  diffexpGenes,
+  world,
+  crossfilter
+) {
+  crossfilter = userDefinedGenes.reduce(
+    (xflt, gene) =>
+      xflt.addDimension(
+        userDefinedDimensionName(gene),
+        "scalar",
+        world.varData.col(gene).asArray(),
+        Float32Array
+      ),
+    crossfilter
+  );
+  crossfilter = diffexpGenes.reduce(
+    (xflt, gene) =>
+      xflt.addDimension(
+        diffexpDimensionName(gene),
+        "scalar",
+        world.varData.col(gene).asArray(),
+        Float32Array
+      ),
+    crossfilter
+  );
+  return crossfilter;
+}
+
+export function pruneVarDataCache(varData, needed) {
+  /*
+  Remove any unneeded columns from the varData dataframe.  Will only
+  prune / remove if the total column count exceeds VarDataCacheLowWatermark
+
+  Note: this code leverages the fact that dataframe offsets indicate
+  the order in which the columns were added.  This crudely provides
+  LRU semantics, so we can delete "older" columns first.
+  */
+
+  /*
+  VarDataCacheLowWatermark - this cofig value sets the minimum cache size,
+  in columns, below which we don't throw away data.
+
+  The value should be high enough so we are caching the maximum which will
+  "typically" be used in the UI (currently: 10 for diffexp, and N for user-
+  specified genes), and low enough to account for memory use (any single
+  column size is 4 bytes * numObs, so a column can be multi-megabyte in common
+  use cases).
+  */
+  const VarDataCacheLowWatermark = 32;
+
+  const numOverWatermark = varData.dims[1] - VarDataCacheLowWatermark;
+  if (numOverWatermark <= 0) return varData;
+
+  const { colIndex } = varData;
+  const all = colIndex.keys();
+  const unused = _.difference(all, needed);
+  if (unused.length > 0) {
+    // sort by offset in the dataframe - ie, psuedo-LRU
+    unused.sort((a, b) => colIndex.getOffset(a) - colIndex.getOffset(b));
+    const numToDrop =
+      unused.length < numOverWatermark ? unused.length : numOverWatermark;
+    for (let i = 0; i < numToDrop; i += 1) {
+      varData = varData.dropCol(unused[i]);
+    }
+  }
+  return varData;
+}

client/src/util/stateManager/index.js

@@ -14,7 +14,8 @@ This is all VERY tightly integrated with reducers and actions, and
 exists to support those concepts.
 */
 
+export { default as createColors } from "./colorHelpers";
 export * as Universe from "./universe";
 export * as World from "./world";
-export * as kvCache from "./keyvalcache";
 export * as WorldUtil from "./worldUtil";
+export * as ControlsHelpers from "./controlsHelpers";

client/src/util/stateManager/keyvalcache.js

@@ -1,122 +0,0 @@
-// jshint esversion: 6
-import _ from "lodash";
-
-/*
-Very simple key/value cache for use by World & Universe.   Cache keys must
-be a string, and values are any JS non-primitive value.
-
-  * constructor(lowWatermark, minTTL):
-      - lowWatermark defines the number of cache elements below which
-        flushing will not occur.
-      - minTTL defines minimum time in milliseconds that cache entries will live.
-        A value of -1 disables automatic flushing (flush() can still
-        be called by external user).
-  * set() - add a key/val pair.
-  * get() - get a value or undefined if not present.
-  * flush(minAgeMs) - flush cache entries in excess of lowWatermark if those
-    entries are older than minAgeMs.
-
-*/
-
-const cachePrivateKey = "__kvcachekey__";
-const defaultLowWatermark = 32;
-const defaultMinTTL = 1000;
-
-function create(lowWatermark = defaultLowWatermark, minTTL = defaultMinTTL) {
-  if (typeof minTTL !== "number" || typeof lowWatermark !== "number") {
-    throw new TypeError(
-      "minTTL and lowWatermark parameters must be a primitive number"
-    );
-  }
-  if (lowWatermark < 0 || minTTL < 0) {
-    throw new RangeError(
-      "minTTL and lowWatermark parameters must be number greater than zero"
-    );
-  }
-
-  return {
-    [cachePrivateKey]: {
-      lowWatermark,
-      minTTL
-    }
-  };
-}
-
-function get(kvcache, key) {
-  if (key === cachePrivateKey) {
-    throw new RangeError(`key parameter may not have value ${cachePrivateKey}`);
-  }
-
-  const val = kvcache[key];
-  if (val) {
-    val[cachePrivateKey] = Date.now();
-  }
-  return val;
-}
-
-function set(kvcache, key, val) {
-  if (key === cachePrivateKey) {
-    throw new RangeError(`key parameter may not have value ${cachePrivateKey}`);
-  }
-
-  const newKvCache = { ...kvcache };
-  newKvCache[key] = val;
-  val[cachePrivateKey] = Date.now();
-  flushInPlace(newKvCache);
-  return newKvCache;
-}
-
-function flush(kvcache) {
-  const newKvCache = { ...kvcache };
-  flushInPlace(newKvCache);
-  return newKvCache;
-}
-
-/*
-Flush elements from cache IF cache size is greater than lowWatermark, and
-those elements are older than minAgeMS
-*/
-function flushInPlace(kvCache) {
-  const { lowWatermark, minTTL } = kvCache[cachePrivateKey];
-  const eol = Date.now() - minTTL;
-  const allKeys = _(kvCache)
-    .keys()
-    .filter(k => k !== cachePrivateKey)
-    .sortBy([k => kvCache[k][cachePrivateKey]])
-    .value();
-
-  if (allKeys.length > lowWatermark) {
-    const keysToDelete = _(allKeys)
-      .slice(0, allKeys.length - lowWatermark)
-      .filter(k => kvCache[k][cachePrivateKey] <= eol)
-      .value();
-    _.forEach(keysToDelete, k => delete kvCache[k]);
-  }
-
-  return kvCache;
-}
-
-/*
-use to create a cache that is a transformation of another cache.
-*/
-function map(srcKvCache, cb, createOptions) {
-  const keysInSrcKvCache = _(srcKvCache)
-    .keys()
-    .filter(k => k !== cachePrivateKey)
-    .value();
-  const lowWatermark = _.get(
-    createOptions,
-    "lowWatermark",
-    defaultLowWatermark
-  );
-  const minTTL = _.get(createOptions, "minTTL", defaultMinTTL);
-  const newKvCache = create(lowWatermark, minTTL);
-  _.forEach(keysInSrcKvCache, key => {
-    const val = cb(get(srcKvCache, key), key);
-    newKvCache[key] = val;
-    val[cachePrivateKey] = Date.now();
-  });
-  return newKvCache;
-}
-
-export { create, get, set, flush, map };

client/src/util/stateManager/summarizeAnnotations.js

@@ -1,122 +0,0 @@
-import _ from "lodash";
-import finiteExtent from "../finiteExtent";
-
-/*
-Build and return obs/var summary using any annotation in the schema
-
-Summary information for each annotation, keyed by annotation name.
-Value will be an object, containing summary information.
-
-For continuous annotations (int, float, etc):
-  <annotation_name>: {
-    categorical: false,
-    range {
-      min: <number>,
-      max: <number>
-    }
-  }
-
-For categorical annotations (boolean, string, category):
-  <annotation_name>: {
-    categorical: true,
-    categories: [ <category1>, <category2>, ... ]
-    categoryCounts: Map {
-      <category1>: <number>,
-      ...
-    },
-    numCategories: <number>
-  }
-
-Summarize will be returned for BOTH obs and var annotations.
-
-Example:
-  {
-    "Splice_sites_Annotated": {
-      categorical: false,
-      range: {
-        "min": 26,
-        "max": 1075869
-      }
-    },
-    "Selection": {
-      categorical: true,
-      numCategories, 3,
-      categories: [ "Astrocytes(HEPACAM)", "Endothelial(BSC)", "Unpanned" ],
-      categoryCounts: Map {
-        "Astrocytes(HEPACAM)": 714,
-        "Endothelial(BSC)": 123,
-        "Unpanned": 665
-      }
-    }
-  }
-
-NOTE: will not summarize the required 'name' annotation, as that is
-specified as unique per element.
-*/
-function _summarizeAnnotations(_schema, annotations) {
-  const summary = _(_schema) // lodash wrapping: https://lodash.com/docs/4.17.11#lodash
-    .filter(v => v.name !== "name")
-    .keyBy("name")
-    .mapValues(anno => {
-      const { name, type } = anno;
-      const continuous = type === "int32" || type === "float32";
-
-      if (continuous) {
-        let min;
-        let max;
-        let nan = 0;
-        let pinf = 0;
-        let ninf = 0;
-        for (let r = 0; r < annotations.length; r += 1) {
-          const val = Number(annotations[r][name]);
-          if (Number.isFinite(val)) {
-            if (min === undefined) {
-              min = val;
-              max = val;
-            } else {
-              min = val < min ? val : min;
-              max = val > max ? val : max;
-            }
-          } else if (Number.isNaN(val)) {
-            nan += 1;
-          } else if (val > 0) {
-            pinf += 1;
-          } else {
-            ninf += 1;
-          }
-        }
-        return {
-          categorical: false,
-          range: { min, max, nan, pinf, ninf }
-        };
-      }
-
-      /* else categorical */
-      const categoryCounts = new Map();
-      for (let r = 0; r < annotations.length; r += 1) {
-        const val = annotations[r][name];
-        let curCount = categoryCounts.get(val);
-        if (curCount === undefined) curCount = 0;
-        categoryCounts.set(val, curCount + 1);
-      }
-      return {
-        categorical: true,
-        categories: [...categoryCounts.keys()],
-        categoryCounts,
-        numCategories: categoryCounts.size
-      };
-    })
-    .value();
-  return summary;
-}
-
-export default function summarizeAnnotations(
-  schema,
-  obsAnnotations,
-  varAnnotations
-) {
-  return {
-    obs: _summarizeAnnotations(schema.annotations.obs, obsAnnotations),
-    var: _summarizeAnnotations(schema.annotations.var, varAnnotations)
-  };
-}

client/src/util/stateManager/universe.js

@@ -2,24 +2,15 @@
 
 import _ from "lodash";
 
-import * as kvCache from "./keyvalcache";
-import summarizeAnnotations from "./summarizeAnnotations";
 import decodeMatrixFBS from "./matrix";
+import * as Dataframe from "../dataframe";
 
 /*
 Private helper function - create and return a template Universe
 */
 function templateUniverse() {
   /* default universe template */
-
-  /* varDataCache config - see kvCache for semantics */
-  const VarDataCacheLowWatermark = 32; // cache element count
-  const VarDataCacheTTLMs = 1000; // min cache time in MS
-
   return {
-    api: null,
-    finalized: false, // XXX: may not be needed
-
     nObs: 0,
     nVar: 0,
     schema: {},
@@ -27,21 +18,14 @@ function templateUniverse() {
     /*
     Annotations
     */
-    obsAnnotations: [] /* all obs annotations, by obs index */,
-    varAnnotations: [] /* all var annotations, by var index */,
-    obsNameToIndexMap: {} /* reverse map 'name' to index */,
-    varNameToIndexMap: {} /* reverse map 'name' to index */,
-    summary: null /* derived data summaries XXX: consider exploding in place */,
-
-    obsLayout: { X: [], Y: [] } /* xy layout */,
+    obsAnnotations: Dataframe.Dataframe.empty(),
+    varAnnotations: Dataframe.Dataframe.empty(),
+    obsLayout: Dataframe.Dataframe.empty(),
 
     /*
-    Cache of var data (expression), by var annotation name.   Data can be
-    accesses as a POJO, but if you want caching semantics, use the kvCache
-    API (eg., kvCache.get(), kvCache.set(), ...), which will maintain the
-    LRU semantics.
+    Var data columns - subset of all
     */
-    varDataCache: kvCache.create(VarDataCacheLowWatermark, VarDataCacheTTLMs)
+    varData: Dataframe.Dataframe.empty(null, new Dataframe.KeyIndex())
   };
 }
 
@@ -53,81 +37,29 @@ These functions are used exclusively by the actions and reducers to
 build an internal POJO for use by the rendering components.
 */
 
-/*
-generate any client-side transformations or summarization that
-is independent of REST API response formats.
-*/
-function finalize(universe) {
-  /* A bit of sanity checking! */
-  const { nObs, nVar } = universe;
-  if (
-    nObs !== universe.obsAnnotations.length ||
-    nObs !== universe.obsLayout.X.length ||
-    nObs !== universe.obsLayout.Y.length ||
-    nVar !== universe.varAnnotations.length
-  ) {
-    throw new Error("Universe dimensionality mismatch - failed to load");
-  }
-  // TODO: add more sanity checks, such as:
-  //  - all annotations in the schema
-  //  - layout has supported number of dimensions
-  //  - ...
-
+function AnnotationsFBSToDataframe(arrayBuffer) {
   /*
-  Create all derived (convenience) data structures.
-  */
-  universe.obsNameToIndexMap = _.transform(
-    universe.obsAnnotations,
-    (acc, value, idx) => {
-      acc[value.name] = idx;
-    },
-    {}
-  );
-  universe.varNameToIndexMap = _.transform(
-    universe.varAnnotations,
-    (acc, value, idx) => {
-      acc[value.name] = idx;
-    },
-    {}
-  );
-  universe.finalized = true;
-  return universe;
-}
-
-function RESTv02AnotationsFBSResponseToInternal(arrayBuffer) {
-  /*
-  Convert a Matrix FBS to our internal format -- row-major array of
-  observations/cells, stored as an object.  Each obs has a key for each
-  annotation, plus __index__ containing its obsIndex.
-
-  Example:
-  [
-    { __index__: 0, tissue_type: "lung", sex: "F", ... },
-    ...
-  ]
-
-  XXX TODO: we could make use of the columns in building crossfilter
-  dimensions (they have to be recreated).  Future optimization.
+  Convert a Matrix FBS to a Dataframe.
   */
   const fbs = decodeMatrixFBS(arrayBuffer);
-  const keys = fbs.colIdx;
-  const result = Array(fbs.nRows);
-  for (let row = 0; row < fbs.nRows; row += 1) {
-    const rec = { __index__: row };
-    for (let col = 0; col < fbs.nCols; col += 1) {
-      rec[keys[col]] = fbs.columns[col][row];
-    }
-    result[row] = rec;
-  }
-  return result;
+  const df = new Dataframe.Dataframe(
+    [fbs.nRows, fbs.nCols],
+    fbs.columns,
+    null,
+    new Dataframe.KeyIndex(fbs.colIdx)
+  );
+  return df;
 }
 
-function RESTv02LayoutFBSResponseToInternal(arrayBuffer) {
+function LayoutFBSToDataframe(arrayBuffer) {
   const fbs = decodeMatrixFBS(arrayBuffer, true);
-  return {
-    X: fbs.columns[0],
-    Y: fbs.columns[1]
-  };
+  const df = new Dataframe.Dataframe(
+    [fbs.nRows, fbs.nCols],
+    fbs.columns,
+    null,
+    new Dataframe.KeyIndex(["X", "Y"])
+  );
+  return df;
 }
 
 function reconcileSchemaCategoriesWithSummary(universe) {
@@ -149,14 +81,14 @@ function reconcileSchemaCategoriesWithSummary(universe) {
     ) {
       const categories = _.union(
         _.get(s, "categories", []),
-        _.get(universe.summary.obs[s.name], "categories", [])
+        _.get(universe.obsAnnotations.col(s.name).summarize(), "categories", [])
       );
       s.categories = categories;
     }
   });
 }
 
-export function createUniverseFromRestV02Response(
+export function createUniverseFromResponse(
   configResponse,
   schemaResponse,
   annotationsObsResponse,
@@ -169,33 +101,28 @@ export function createUniverseFromRestV02Response(
   const { schema } = schemaResponse;
   const universe = templateUniverse();
 
-  /* constants */
-  universe.api = "0.2";
-
   /* schema related */
   universe.schema = schema;
   universe.nObs = schema.dataframe.nObs;
   universe.nVar = schema.dataframe.nVar;
 
   /* annotations */
-  universe.obsAnnotations = RESTv02AnotationsFBSResponseToInternal(
-    annotationsObsResponse
-  );
-  universe.varAnnotations = RESTv02AnotationsFBSResponseToInternal(
-    annotationsVarResponse
-  );
-
+  universe.obsAnnotations = AnnotationsFBSToDataframe(annotationsObsResponse);
+  universe.varAnnotations = AnnotationsFBSToDataframe(annotationsVarResponse);
   /* layout */
-  universe.obsLayout = RESTv02LayoutFBSResponseToInternal(layoutFBSResponse);
+  universe.obsLayout = LayoutFBSToDataframe(layoutFBSResponse);
 
-  universe.summary = summarizeAnnotations(
-    universe.schema,
-    universe.obsAnnotations,
-    universe.varAnnotations
-  );
+  /* sanity check */
+  if (
+    universe.nObs !== universe.obsLayout.length ||
+    universe.nObs !== universe.obsAnnotations.length ||
+    universe.nVar !== universe.varAnnotations.length
+  ) {
+    throw new Error("Universe dimensionality mismatch - failed to load");
+  }
 
   reconcileSchemaCategoriesWithSummary(universe);
-  return finalize(universe);
+  return universe;
 }
 
 export function convertDataFBStoObject(universe, arrayBuffer) {
@@ -214,8 +141,8 @@ export function convertDataFBStoObject(universe, arrayBuffer) {
   const result = {};
 
   for (let c = 0; c < colIdx.length; c += 1) {
-    const gene = universe.varAnnotations[colIdx[c]].name;
-    result[gene] = columns[c];
+    const varName = universe.varAnnotations.at(colIdx[c], "name");
+    result[varName] = columns[c];
   }
   return result;
 }

client/src/util/stateManager/world.js

@@ -1,13 +1,10 @@
 // jshint esversion: 6
 
-import _ from "lodash";
-import * as kvCache from "./keyvalcache";
-import summarizeAnnotations from "./summarizeAnnotations";
 import { layoutDimensionName, obsAnnoDimensionName } from "../nameCreators";
-import Crossfilter from "../typedCrossfilter";
-import { sliceByIndex } from "../typedCrossfilter/util";
+import * as Dataframe from "../dataframe";
 
 /*
+
 World is a subset of universe.   Most code should use world, and should
 (generally) not use Universe.   World contains any per-obs or per-var data
 that must be consistent acorss the app when we view/manipulate subsets
@@ -16,120 +13,76 @@ of Universe.
 Private API indicated by leading underscore in key name (eg, _foo).  Anything else
 is public.
 
-World contains several public keys, obsAnnotations, and obsLayout, which are
-arrays contianing information about an OBS in the same order/offset.  In
-other words, world.obsAnnotations[0] and world.obsLayout.X[0] refer to the same
-obs/cell.
+Notable keys in the world object:
+
+* nObs, nVar: dimensions
+
+* schema: data schema from the server
 
 * obsAnnotations:
 
-  obsAnnotations will return an array of objects.  Each object contains all annotation
-  values for a given observation/cell, keyed by annotation name, PLUS a key
-  '__cellId__', containing a REST API ID for this obs/cell (referred to as the
-  obsIndex in the REST 0.2 spec or cellIndex in the 0.1 spec.
+  Dataframe containing obs annotations.  Columns are indexed by annotation
+  name (eg, 'tissue type'), and rows are indexed by the REST API obsIndex
+  (ie, the offset into the underlying server-side dataframe).
 
-  Example:  [ { __cellId__: 99, cluster: 'blue', numReads: 93933 } ]
-
-  NOTE: world.obsAnnotation should be identical to the old state.cells value,
-  EXCEPT that
-    * __cellIndex__ renamed to __index__
-    * __x__ and __y__ are now in world.obsLayout
-    * __color__ and __colorRBG__ should be moved to controls reducer
+  This indexing means that you can access data by _either_ the server's
+  obxIndex, or the offset into the client-side column array .  Be careful
+  to know which you want and are using.
 
 * obsLayout:
 
-  obsLayout will return an object containing two arrays, containing X and Y
-  coordinates respectively.
+  A dataframe containing the X/Y layout for all obs.  Columns are named
+  'X' and 'Y', and rows are indexed in the same way as obsAnnotation.
 
-  Example: { X: [ 0.33, 0.23, ... ], Y: [ 0.8, 0.777, ... ]}
-
-* crossfilter - a crossfilter object across world.obsAnnotations
-
-* dimensionMap - an object mapping annotation names to dimensions on
-  the crossfilter
+* varData: a cache of expression columns, stored in a Dataframe.  Cache
+  managed by controls reducer.
 
 */
 
-/* varDataCache config - see kvCache for semantics */
-const VarDataCacheLowWatermark = 32; // cache element count
-const VarDataCacheTTLMs = 1000; // min cache time in MS
-
 function templateWorld() {
   return {
-    // map from universe obsIndex to world offset.
-    // Undefined / null indicates identity mapping.
-    obsIndex: null,
-    obsBackIndex: null,
-
     /* schema/version related */
-    api: null,
     schema: null,
     nObs: 0,
     nVar: 0,
 
     /* annotations */
-    obsAnnotations: null,
-    varAnnotations: null,
+    obsAnnotations: Dataframe.Dataframe.empty(),
+    varAnnotations: Dataframe.Dataframe.empty(),
 
-    /* layout of graph */
-    obsLayout: null,
+    /* layout of graph. Dataframe. */
+    obsLayout: Dataframe.Dataframe.empty(),
 
-    /* derived data summaries XXX: consider exploding in place */
-    summary: null,
-
-    varDataCache: kvCache.create(
-      VarDataCacheLowWatermark,
-      VarDataCacheTTLMs
-    ) /* cache of var data (expression) */
+    /*
+    Var data columns - subset of all data (may be empty)
+    */
+    varData: Dataframe.Dataframe.empty(null, new Dataframe.KeyIndex())
   };
 }
 
 export function createWorldFromEntireUniverse(universe) {
-  if (!universe.finalized) {
-    throw new Error("World can't be created from an partial Universe");
-  }
-
   const world = templateWorld();
 
-  // map from the universe obsIndex to our world offset.
-  // undefined/null indicates identity map.
-  // In other words   obsBackIndex[universeIdx] -> worldIdx
-  world.obsBackIndex = null;
-  // Map to the universe index for each element in world.
-  // Null indicates identity map (aka world === universe)
-  // In other wrods  obsIndex[worldIdx]  ->  universeIdx
-  world.obsIndex = null;
-
   /*
   public interface follows
   */
 
   /* Schema related */
-  world.api = universe.api;
   world.schema = universe.schema;
   world.nObs = universe.nObs;
   world.nVar = universe.nVar;
 
-  /* annotations */
+  /* annotation dataframes */
   world.obsAnnotations = universe.obsAnnotations;
   world.varAnnotations = universe.varAnnotations;
 
-  /* layout and display characteristics */
+  /* layout and display characteristics dataframe */
   world.obsLayout = universe.obsLayout;
 
-  /* derived data & summaries */
-  world.summary = summarizeAnnotations(
-    world.schema,
-    world.obsAnnotations,
-    world.varAnnotations
-  );
-
-  /* build the varDataCache */
-  world.varDataCache = kvCache.map(
-    universe.varDataCache,
-    val => subsetVarData(world, universe, val),
-    { lowWatermark: VarDataCacheLowWatermark, minTTL: VarDataCacheTTLMs }
-  );
+  /*
+  Var data columns - subset of all
+  */
+  world.varData = universe.varData.clone();
 
   return world;
 }
@@ -138,54 +91,24 @@ export function createWorldFromCurrentSelection(universe, world, crossfilter) {
   const newWorld = templateWorld();
 
   /* these don't change as only OBS are selected in our current implementation */
-  newWorld.api = universe.api;
   newWorld.nVar = universe.nVar;
   newWorld.schema = universe.schema;
   newWorld.varAnnotations = universe.varAnnotations;
 
-  /* build index maps and back maps based upon current selection state */
-  const obsBackIndex = new Uint32Array(universe.nObs);
-  obsBackIndex.fill(-1); // default - aka unused
-  const notSelected = obsBackIndex[0];
-  let nObs = 0;
-  for (let i = 0; i < universe.nObs; i += 1) {
-    if (crossfilter.isElementFiltered(i)) {
-      obsBackIndex[i] = nObs;
-      nObs += 1;
-    }
+  /* now subset/cut obs */
+  const mask = crossfilter.allSelectedMask();
+  newWorld.obsAnnotations = world.obsAnnotations.isubsetMask(mask);
+  newWorld.obsLayout = world.obsLayout.isubsetMask(mask);
+  newWorld.nObs = newWorld.obsAnnotations.dims[0];
+
+  /*
+  Var data columns - subset of all
+  */
+  if (world.varData.isEmpty()) {
+    newWorld.varData = world.varData.clone();
+  } else {
+    newWorld.varData = world.varData.isubsetMask(mask);
   }
-  const obsIndex = new Uint32Array(nObs);
-  for (let i = 0; i < universe.nObs; i += 1) {
-    const worldIdx = obsBackIndex[i];
-    if (worldIdx !== notSelected) {
-      obsIndex[worldIdx] = i;
-    }
-  }
-
-  newWorld.nObs = nObs;
-  newWorld.obsIndex = obsIndex;
-  newWorld.obsBackIndex = obsBackIndex;
-
-  /* now slice */
-  newWorld.obsAnnotations = sliceByIndex(universe.obsAnnotations, obsIndex);
-  newWorld.obsLayout = {
-    X: sliceByIndex(universe.obsLayout.X, obsIndex),
-    Y: sliceByIndex(universe.obsLayout.Y, obsIndex)
-  };
-
-  /* derived data & summaries */
-  newWorld.summary = summarizeAnnotations(
-    newWorld.schema,
-    newWorld.obsAnnotations,
-    newWorld.varAnnotations
-  );
-
-  /* build the varDataCache */
-  newWorld.varDataCache = kvCache.map(
-    universe.varDataCache,
-    val => subsetVarData(newWorld, universe, val),
-    { lowWatermark: VarDataCacheLowWatermark, minTTL: VarDataCacheTTLMs }
-  );
   return newWorld;
 }
 
@@ -214,79 +137,52 @@ function deduceDimensionType(attributes, fieldName) {
   return dimensionType;
 }
 
-/*
-  Return a crossfilter dimension for the specified world & named gene.
-
-  NOTE: this assumes that the expression data was already loaded,
-  by calling an appropriate action creator.
-
-  Caller needs to *save* this dimension somewhere for it to be later used.
-  Dimension must be destroyed by calling dimension.dispose()
-  when it is no longer needed
-  (it will not be garbage collected without this call)
-*/
-
-export function createVarDimension(
-  world,
-  _worldVarDataCache,
-  crossfilter,
-  geneName
-) {
-  // return crossfilter.dimension(_worldVarDataCache[geneName], Float32Array);
-  return crossfilter.dimension(
-    Crossfilter.ScalarDimension,
-    _worldVarDataCache[geneName],
-    Float32Array
-  );
-}
-
-export function createObsDimensionMap(crossfilter, world) {
+export function createObsDimensions(crossfilter, world) {
   /*
-  create and return a crossfilter dimension for every obs annotation
-  for which we have a supported type.
+  create and return a crossfilter with a dimension for every obs annotation
+  for which we have a supported type, *except* 'name'
   */
-  const { schema, obsLayout } = world;
+  const { schema, obsLayout, obsAnnotations } = world;
+  const annoList = schema.annotations.obs.filter(anno => anno.name !== "name");
+  crossfilter = annoList.reduce((xfltr, anno) => {
+    const dimType = deduceDimensionType(anno, anno.name);
+    const colData = obsAnnotations.col(anno.name).asArray();
+    const name = obsAnnoDimensionName(anno.name);
+    if (dimType === "enum") {
+      return xfltr.addDimension(name, "enum", colData);
+    }
+    if (dimType) {
+      return xfltr.addDimension(name, "scalar", colData, dimType);
+    }
+    return xfltr;
+  }, crossfilter);
 
-  // Create a crossfilter dimension for all obs annotations *except* 'name'
-  const dimensionMap = _(schema.annotations.obs)
-    .filter(anno => anno.name !== "name")
-    .transform((result, anno) => {
-      const dimType = deduceDimensionType(anno, anno.name);
-      if (dimType === "enum") {
-        result[obsAnnoDimensionName(anno.name)] = crossfilter.dimension(
-          Crossfilter.EnumDimension,
-          r => r[anno.name]
-        );
-      } else {
-        result[obsAnnoDimensionName(anno.name)] = crossfilter.dimension(
-          Crossfilter.ScalarDimension,
-          r => r[anno.name],
-          dimType
-        );
-      } // else ignore the annotation
-    }, {})
-    .value();
-
-  /*
-  Add crossfilter dimensions allowing filtering on layout
-  */
-  dimensionMap[layoutDimensionName("XY")] = crossfilter.dimension(
-    Crossfilter.SpatialDimension,
-    obsLayout.X,
-    obsLayout.Y
+  return crossfilter.addDimension(
+    layoutDimensionName("XY"),
+    "spatial",
+    obsLayout.col("X").asArray(),
+    obsLayout.col("Y").asArray()
   );
-
-  return dimensionMap;
 }
 
 export function worldEqUniverse(world, universe) {
   return world.obsAnnotations === universe.obsAnnotations;
 }
 
-export function subsetVarData(world, universe, varData) {
-  // If world === universe, just return the entire varData array
-  if (worldEqUniverse(world, universe)) {
-    return varData;
+export function getSelectedByIndex(crossfilter) {
+  /*
+  return array of obsIndex, containing all selected obs/cells.
+  */
+  const selected = crossfilter.allSelectedMask(); // array of bool-ish
+  const keys = crossfilter.data.rowIndex.keys(); // row keys, aka universe rowIndex
+
+  const set = new Int32Array(selected.length);
+  let numElems = 0;
+  for (let i = 0, l = selected.length; i < l; i += 1) {
+    if (selected[i]) {
+      set[numElems] = keys[i];
+      numElems += 1;
+    }
   }
-  return sliceByIndex(varData, world.obsIndex);
+  return new Int32Array(set.buffer, 0, numElems);
 }

client/src/util/stateManager/worldUtil.js

@@ -18,13 +18,23 @@ Map {
    ...
 }
 
+Parameters are:
+- dim1: dimension 1 name/label
+- dim2: dimension 2 name/label
+- df: dataframe containing dim1 and dim2 on the column axis
+
 */
-function _countCategoryValues2D(dim1, dim2, rows) {
+function _countCategoryValues2D(dim1, dim2, df) {
   const dimMap = new Map();
-  for (let r = 0; r < rows.length; r += 1) {
-    const row = rows[r];
-    const val1 = row[dim1];
-    const val2 = row[dim2];
+  const col1 = df.col(dim1) ? df.col(dim1).asArray() : null;
+  const col2 = df.col(dim2) ? df.col(dim2).asArray() : null;
+  if (!col1 || !col2) {
+    return dimMap;
+  }
+
+  for (let r = 0, l = df.length; r < l; r += 1) {
+    const val1 = col1[r];
+    const val2 = col2[r];
     let d2Map = dimMap.get(val1);
     if (d2Map === undefined) {
       d2Map = new Map();

client/src/util/typedCrossfilter/bitArray.js

@@ -40,7 +40,7 @@ class BitArray {
   // Return the number of records that are selected, ie, have a one bit in
   // all allocated dimensions.
   //
-  get selectionCount() {
+  selectionCount() {
     return this.countAllOnes();
   }
 
@@ -48,16 +48,27 @@ class BitArray {
   //
   countAllOnes() {
     let count = 0;
-    const { bitarray, bitmask, length, width } = this;
-    for (let l = 0; l < length; l += 1) {
-      let dimensionsSet = 0;
-      for (let w = 0; w < width; w += 1) {
-        if (bitarray[w * length + l] === bitmask[w]) {
-          dimensionsSet += 1;
+    const { bitarray, length, width } = this;
+    if (width === 1) {
+      // special case, width === 1, for performance
+      const bitmask = this.bitmask[0];
+      for (let l = 0; l < length; l += 1) {
+        if (bitarray[l] === bitmask) {
+          count += 1;
         }
       }
-      if (dimensionsSet === width) {
-        count += 1;
+    } else {
+      const { bitmask } = this;
+      for (let l = 0; l < length; l += 1) {
+        let dimensionsSet = 0;
+        for (let w = 0; w < width; w += 1) {
+          if (bitarray[w * length + l] === bitmask[w]) {
+            dimensionsSet += 1;
+          }
+        }
+        if (dimensionsSet === width) {
+          count += 1;
+        }
       }
     }
     return count;
@@ -200,6 +211,19 @@ class BitArray {
     }
   }
 
+  // select range of indices on a dimension
+  //
+  selectFromRange(dim, range) {
+    const col = dim >>> 5;
+    const first = range[0];
+    const last = range[1];
+    const one = 1 << dim % 32;
+    const offset = col * this.length;
+    for (let i = first; i < last; i += 1) {
+      this.bitarray[offset + i] |= one;
+    }
+  }
+
   // select range of indices on a dimension, indirect through a sort map.
   // Indirect functions are used to map between sort and natural order.
   //
@@ -214,6 +238,19 @@ class BitArray {
     }
   }
 
+  // deselect range of indices on a dimension
+  //
+  deselectFromRange(dim, range) {
+    const col = dim >>> 5;
+    const first = range[0];
+    const last = range[1];
+    const zero = ~(1 << dim % 32);
+    const offset = col * this.length;
+    for (let i = first; i < last; i += 1) {
+      this.bitarray[offset + i] &= zero;
+    }
+  }
+
   // deselect range of indices on a dimension, indirect through a sort map.
   //
   deselectIndirectFromRange(dim, indirect, range) {
@@ -233,12 +270,14 @@ class BitArray {
   fillBySelection(result, selectedValue, deselectedValue) {
     // special case (width === 1) for performance
     if (this.width === 1) {
-      const bitmask = this.bitmask[0];
-      for (let i = 0, len = this.length; i < len; i += 1) {
-        result[i] =
-          bitmask && this.bitarray[i] === bitmask
-            ? selectedValue
-            : deselectedValue;
+      const { bitmask, bitarray } = this;
+      const mask = bitmask[0];
+      if (!mask) {
+        result.fill(deselectedValue);
+      } else {
+        for (let i = 0, len = this.length; i < len; i += 1) {
+          result[i] = bitarray[i] === mask ? selectedValue : deselectedValue;
+        }
       }
     } else {
       for (let i = 0, len = this.length; i < len; i += 1) {

client/src/util/typedCrossfilter/crossfilter.js

@@ -0,0 +1,594 @@
+import { polygonContains } from "d3";
+
+import PositiveIntervals from "./positiveIntervals";
+import BitArray from "./bitArray";
+import { sort } from "./sort";
+import {
+  makeSortIndex,
+  lowerBound,
+  lowerBoundIndirect,
+  upperBoundIndirect
+} from "./util";
+
+class NotImplementedError extends Error {
+  constructor(...params) {
+    super(...params);
+
+    // Maintains proper stack trace for where our error was thrown (only available on V8)
+    if (Error.captureStackTrace) {
+      Error.captureStackTrace(this, NotImplementedError);
+    }
+  }
+}
+
+export default class ImmutableTypedCrossfilter {
+  constructor(data, dimensions = {}, selectionCache = null) {
+    /*
+    Typically, parameter 'data' is one of:
+      - Array of objects/records
+      - Dataframe (util/dataframe)
+    Other parameters are only used internally.
+
+    Object field description:
+    - data: reference to the array of records in the crossfilter
+    - selectionBitArray: bit array containing the flatted selection state
+      of all dimensions.  This is lazily created and is effectively
+      a perfomance cache.  Methods which return a new crossfilter,
+      such as select(), addDimention() and delDimension(), will pass
+      the cache forward to the new object, as the typical "immutable API"
+      usage pattern is to retain the new crossfilter and discard the old.
+    - dimensions: contains each dimension and its current state:
+        - id: bit offset in the cached bit array
+        - dim: the dimension object
+        - name: the dimension name
+        - selection: the dimension's current selection
+    */
+    this.data = data;
+    this.selectionCache = selectionCache; /* BitArray */
+    this.dimensions = dimensions; /* name: { id, dim, name, selection } */
+  }
+
+  size() {
+    return this.data.length;
+  }
+
+  all() {
+    return this.data;
+  }
+
+  dimensionNames() {
+    /* return array of all dimensions (by name) */
+    return Object.keys(this.dimensions);
+  }
+
+  addDimension(name, type, ...rest) {
+    /*
+    Add a new dimension to this crossfilter, of type DimensionType.
+    Remainder of parameters are dimension-type-specific.
+    */
+    const { data, selectionCache } = this;
+
+    if (this.dimensions[name] !== undefined) {
+      throw new Error(`Adding duplicate dimension name ${name}`);
+    }
+
+    this.selectionCache = null; // pass ownership to new crossfilter
+
+    let id;
+    if (selectionCache) {
+      id = selectionCache.allocDimension();
+      selectionCache.selectAll(id);
+    }
+    const DimensionType = DimTypes[type];
+    const dim = new DimensionType(name, data, ...rest);
+    const dimensions = {
+      ...this.dimensions,
+      [name]: {
+        id,
+        dim,
+        name,
+        selection: dim.select({ mode: "all" })
+      }
+    };
+    return new ImmutableTypedCrossfilter(data, dimensions, selectionCache);
+  }
+
+  delDimension(name) {
+    const { data, selectionCache } = this;
+    const dimensions = { ...this.dimensions };
+    if (dimensions[name] === undefined) {
+      throw new ReferenceError(`Unable to delete unknown dimension ${name}`);
+    }
+
+    const { id } = dimensions[name];
+    delete dimensions[name];
+    this.selectionCache = null; // pass ownership to new crossfilter
+    if (selectionCache) {
+      selectionCache.freeDimension(id);
+    }
+    return new ImmutableTypedCrossfilter(data, dimensions, selectionCache);
+  }
+
+  select(name, spec) {
+    /*
+    select on named dimension, as indicated by `spec`.   Spec is an object
+    specifying the selection, and must contain at least a `mode` field.
+
+    Examples:
+      select("foo", {mode: "all"});
+      select("bar", {mode: "none"});
+      select("mumble", {mode: "exact", values: "blue"});
+      select("mumble", {mode: "exact", values: ["red", "green", "blue"]});
+      select("blort", {mode: "range", lo: 0, hi: 999.99});
+    */
+    const { data, selectionCache } = this;
+    this.selectionCache = null;
+    const dimensions = { ...this.dimensions };
+    const { dim, id, selection: oldSelection } = dimensions[name];
+    const newSelection = dim.select(spec);
+    newSelection.ranges = PositiveIntervals.canonicalize(newSelection.ranges);
+    dimensions[name] = { id, dim, name, selection: newSelection };
+    ImmutableTypedCrossfilter._dimSelnHasUpdated(
+      selectionCache,
+      id,
+      newSelection,
+      oldSelection
+    );
+    return new ImmutableTypedCrossfilter(data, dimensions, selectionCache);
+  }
+
+  static _dimSelnHasUpdated(selectionCache, id, newSeln, oldSeln) {
+    /*
+    Selection has updated from oldSeln to newSeln.   Update the
+    bit array if it exists.  If not, we will lazy create it when
+    needed.
+    */
+    if (selectionCache) {
+      /*
+      if both new and old selection use the same index, we can
+      perform an incremental update. If the index changed, we have
+      to do a suboptimal full deselect/select.
+      */
+      let adds;
+      let dels;
+      if (newSeln.index === oldSeln.index) {
+        adds = PositiveIntervals.difference(newSeln.ranges, oldSeln.ranges);
+        dels = PositiveIntervals.difference(oldSeln.ranges, newSeln.ranges);
+      } else {
+        // console.log("suboptimal selection update - index changed");
+        adds = newSeln.ranges;
+        dels = oldSeln.ranges;
+      }
+
+      /*
+      allow dimensions to return selected ranges in either dimension sort
+      order (indirect via index), or in original record order.
+
+      If sort index exists in the dimension, assume sort ordered ranges.
+      */
+      if (oldSeln.index) {
+        dels.forEach(interval =>
+          selectionCache.deselectIndirectFromRange(id, oldSeln.index, interval)
+        );
+      } else {
+        dels.forEach(interval =>
+          selectionCache.deselectFromRange(id, interval)
+        );
+      }
+
+      if (newSeln.index) {
+        adds.forEach(interval =>
+          selectionCache.selectIndirectFromRange(id, newSeln.index, interval)
+        );
+      } else {
+        adds.forEach(interval => selectionCache.selectFromRange(id, interval));
+      }
+    }
+  }
+
+  _getSelectionCache() {
+    if (!this.selectionCache) {
+      // console.log("...rebuilding crossfilter cache...");
+      const selectionCache = new BitArray(this.data.length);
+      Object.keys(this.dimensions).forEach(name => {
+        const { selection } = this.dimensions[name];
+        const id = selectionCache.allocDimension();
+        this.dimensions[name].id = id;
+        const { ranges, index } = selection;
+        ranges.forEach(range => {
+          if (index) {
+            selectionCache.selectIndirectFromRange(id, index, range);
+          } else {
+            selectionCache.selectFromRange(id, range);
+          }
+        });
+      });
+      this.selectionCache = selectionCache;
+    }
+    return this.selectionCache;
+  }
+
+  allSelected() {
+    /*
+    return array of all records currently selected by all dimensions
+    */
+    const selectionCache = this._getSelectionCache();
+    const { data } = this;
+    if (Array.isArray(data)) {
+      const res = [];
+      for (let i = 0, len = data.length; i < len; i += 1) {
+        if (selectionCache.isSelected(i)) {
+          res.push(data[i]);
+        }
+      }
+      return res;
+    }
+    /* else, Dataframe-like */
+    return data.isubsetMask(this.allSelectedMask());
+  }
+
+  allSelectedMask() {
+    /*
+    return Uint8Array containing selection state (truthy/falsey) for each record.
+    */
+    const selectionCache = this._getSelectionCache();
+    return selectionCache.fillBySelection(
+      new Uint8Array(this.data.length),
+      1,
+      0
+    );
+  }
+
+  countSelected() {
+    /*
+    return number of records selected on all dimensions
+    */
+    const selectionCache = this._getSelectionCache();
+    return selectionCache.selectionCount();
+  }
+
+  isElementSelected(i) {
+    /*
+    return truthy/falsey if this record is selected on all dimensions
+    */
+    const selectionCache = this._getSelectionCache();
+    return selectionCache.isSelected(i);
+  }
+
+  fillByIsSelected(array, selectedValue, deselectedValue) {
+    /*
+    fill array with one of two values, based upon selection state.
+    */
+    const selectionCache = this._getSelectionCache();
+    return selectionCache.fillBySelection(
+      array,
+      selectedValue,
+      deselectedValue
+    );
+  }
+}
+
+/*
+Base dimension object.
+
+A Dimension is an index, accessed via a select() method.  The protocol
+for a dimension:
+  - constructor - first param is name, remainder is whatever params are
+    required to initialize the dimension.
+  - select - one and only param is the selection specifier.  Returns an
+    array of record IDs.
+  - name - the dimension name/label.
+*/
+class _ImmutableBaseDimension {
+  constructor(name) {
+    this.name = name;
+  }
+
+  /* eslint-disable class-methods-use-this */
+  select(spec) {
+    const { mode } = spec;
+    if (mode === undefined) {
+      throw new Error("select spec does not contain 'mode'");
+    }
+    throw new Error(`select mode ${mode} not implemented`);
+  }
+  /* eslint-enable class-methods-use-this */
+}
+
+class ImmutableScalarDimension extends _ImmutableBaseDimension {
+  constructor(name, data, value, ValueArrayType) {
+    super(name);
+
+    // Three modes - caller can provide a pre-created value array,
+    // a map function which will create it, or another array which
+    // will used with an identity map function.
+    let array;
+    if (value instanceof ValueArrayType) {
+      // user has provided the final typed array - just use it
+      if (value.length !== data.length) {
+        throw new RangeError(
+          "ScalarDimension values length must equal crossfilter data record count"
+        );
+      }
+      array = value;
+    } else if (value instanceof Function) {
+      // Create value array from user-provided map function.
+      array = this._createValueArray(
+        data,
+        value,
+        new ValueArrayType(data.length)
+      );
+    } else if (isArrayOrTypedArray(value)) {
+      // Create value array from user-provided array.  Typically used
+      // only by enumerated dimensions
+      array = this._createValueArray(
+        data,
+        i => value[i],
+        new ValueArrayType(data.length)
+      );
+    } else {
+      throw new NotImplementedError(
+        "dimension value must be function or value array type"
+      );
+    }
+    this.value = array;
+
+    // create sort index
+    this.index = makeSortIndex(array);
+  }
+
+  /* eslint-disable class-methods-use-this */
+  _createValueArray(data, mapf, array) {
+    // create dimension value array
+    const len = data.length;
+    const larray = array;
+    for (let i = 0; i < len; i += 1) {
+      larray[i] = mapf(i, data);
+    }
+    return larray;
+  }
+  /* eslint-enable class-methods-use-this */
+
+  select(spec) {
+    const { mode } = spec;
+    const { index } = this;
+    switch (mode) {
+      case "all":
+        return { ranges: [[0, this.value.length]], index };
+      case "none":
+        return { ranges: [], index };
+      case "exact":
+        return this.selectExact(spec);
+      case "range":
+        return this.selectRange(spec);
+      default:
+        return super.select(spec);
+    }
+  }
+
+  selectExact(spec) {
+    const { value, index } = this;
+    let { values } = spec;
+    if (!Array.isArray(values)) {
+      values = [values];
+    }
+    const ranges = [];
+    for (let v = 0, len = values.length; v < len; v += 1) {
+      const r = [
+        lowerBoundIndirect(value, index, values[v], 0, value.length),
+        upperBoundIndirect(value, index, values[v], 0, value.length)
+      ];
+      if (r[0] <= r[1]) {
+        ranges.push(r);
+      }
+    }
+    return { ranges, index };
+  }
+
+  selectRange(spec) {
+    const { value, index } = this;
+    /* [lo, hi) */
+    const { lo, hi } = spec;
+    const ranges = [];
+    const r = [
+      lowerBoundIndirect(value, index, lo, 0, value.length),
+      lowerBoundIndirect(value, index, hi, 0, value.length)
+    ];
+    if (r[0] < r[1]) ranges.push(r);
+    return { ranges, index };
+  }
+}
+
+class ImmutableEnumDimension extends ImmutableScalarDimension {
+  constructor(name, data, value) {
+    super(name, data, value, Uint32Array);
+  }
+
+  _createValueArray(data, mapf, array) {
+    const len = data.length;
+    const larray = array;
+
+    // create enumeration table - mapping between the value
+    // and the enum.
+    const s = new Set();
+    for (let i = 0; i < len; i += 1) {
+      s.add(mapf(i, data));
+    }
+    const enumIndex = sort(Array.from(s));
+    this.enumIndex = enumIndex;
+
+    // create dimension value array
+    const enumLen = enumIndex.length;
+    for (let i = 0; i < len; i += 1) {
+      const v = mapf(i, data);
+      const e = lowerBound(enumIndex, v, 0, enumLen);
+      larray[i] = e;
+    }
+    return larray;
+  }
+
+  selectExact(spec) {
+    const { enumIndex } = this;
+    const { values } = spec;
+    return super.selectExact({
+      mode: spec.mode,
+      values: values.map(v => lowerBound(enumIndex, v, 0, enumIndex.length))
+    });
+  }
+
+  /* eslint-disable class-methods-use-this */
+  selectRange() {
+    throw new Error("range selection unsupported on Enumerated dimension");
+  }
+  /* eslint-enable class-methods-use-this */
+}
+
+class ImmutableSpatialDimension extends _ImmutableBaseDimension {
+  constructor(name, data, X, Y) {
+    super(name);
+
+    if (X.length !== Y.length && X.length !== data.length) {
+      throw new RangeError(
+        "SpatialDimension values must have same dimensionality as crossfilter"
+      );
+    }
+    this.X = X;
+    this.Y = Y;
+
+    this.Xindex = makeSortIndex(X);
+    this.Yindex = makeSortIndex(Y);
+  }
+
+  select(spec) {
+    const { mode } = spec;
+    switch (mode) {
+      case "all":
+        return { ranges: [[0, this.X.length]], index: null };
+      case "none":
+        return { ranges: [], index: null };
+      case "within-rect":
+        return this.selectWithinRect(spec);
+      case "within-polygon":
+        return this.selectWithinPolygon(spec);
+      default:
+        return super.select(spec);
+    }
+  }
+
+  selectWithinRect(spec) {
+    /*
+      { mode: "within-rect", x0: 1, y0: 0, x1: 3, y1: 9 }
+    */
+    const { x0, y0, x1, y1 } = spec;
+    const { X, Y } = this;
+    const ranges = [];
+    let start = -1;
+    for (let i = 0, l = X.length; i < l; i += 1) {
+      const x = X[i];
+      const y = Y[i];
+      const inside = x0 <= x && x < x1 && y0 <= y && y < y1;
+      if (inside && start === -1) start = i;
+      if (!inside && start !== -1) {
+        ranges.push([start, i]);
+        start = -1;
+      }
+    }
+    if (start !== -1) ranges.push([start, X.length]);
+    return { ranges, index: null };
+  }
+
+  /*
+  Relatively brute force filter by polygon.
+
+  Currently uses d3.polygonContains() to test for polygon inclusion, which itself
+  uses a ray casting (crossing number) algorithm.  There are a series of optimizations
+  to make this faster:
+    * first sliced by X or Y, using an index on the axis
+    * then the polygon bounding box is used for trivial rejection
+    * then the polygon test is applied
+  */
+
+  selectWithinPolygon(spec) {
+    /*
+      { mode: "within-polygon", polygon: [ [x0, y0], ... ] }
+    */
+    const { polygon } = spec;
+    const [minX, minY, maxX, maxY] = polygonBoundingBox(polygon);
+    const { X, Y, Xindex, Yindex } = this;
+    const { length } = X;
+    let slice;
+    let index;
+    if (maxY - minY > maxX - minX) {
+      slice = [
+        lowerBoundIndirect(X, Xindex, minX, 0, length),
+        lowerBoundIndirect(X, Xindex, maxX, 0, length)
+      ];
+      index = Xindex;
+    } else {
+      slice = [
+        lowerBoundIndirect(Y, Yindex, minY, 0, length),
+        lowerBoundIndirect(Y, Yindex, maxY, 0, length)
+      ];
+      index = Yindex;
+    }
+
+    const ranges = [];
+    let start = -1;
+    for (let i = slice[0], e = slice[1]; i < e; i += 1) {
+      const rid = index[i];
+      const x = X[rid];
+      const y = Y[rid];
+      const inside =
+        minX <= x &&
+        x < maxX &&
+        minY <= y &&
+        y < maxY &&
+        withinPolygon(polygon, x, y);
+
+      if (inside && start === -1) start = i;
+      if (!inside && start !== -1) {
+        ranges.push([start, i]);
+        start = -1;
+      }
+    }
+    if (start !== -1) ranges.push([start, slice[1]]);
+    return { ranges, index };
+  }
+}
+
+/* Helpers */
+export const DimTypes = {
+  scalar: ImmutableScalarDimension,
+  enum: ImmutableEnumDimension,
+  spatial: ImmutableSpatialDimension
+};
+
+function isArrayOrTypedArray(x) {
+  return (
+    Array.isArray(x) ||
+    (ArrayBuffer.isView(x) &&
+      Object.prototype.toString.call(x) !== "[object DataView]")
+  );
+}
+
+/* return bounding box of the polygon */
+function polygonBoundingBox(polygon) {
+  let minX = Number.MAX_VALUE;
+  let minY = Number.MAX_VALUE;
+  let maxX = Number.MIN_VALUE;
+  let maxY = Number.MIN_VALUE;
+  for (let i = 0, l = polygon.length; i < l; i += 1) {
+    const point = polygon[i];
+    const [x, y] = point;
+    if (x < minX) minX = x;
+    if (y < minY) minY = y;
+    if (x > maxX) maxX = x;
+    if (y > maxY) maxY = y;
+  }
+  return [minX, minY, maxX, maxY];
+}
+
+function withinPolygon(polygon, x, y) {
+  // TODO XXX replace
+  return polygonContains(polygon, [x, y]);
+}

client/src/util/typedCrossfilter/index.js

@@ -1,7 +1,5 @@
-// jshint esversion: 6
-
 /*
-Typedarray Crossfilter - a re-implementation of a subset of crossfilter, with
+Crossfilter - a re-implementation of a subset of crossfilter, with
 time/space optimizations predicated upon the following assumptions:
   - dimensions are uniformly typed, and all values must be of that type
   - dimension values must be a primitive type (int, float, string).  Arrays
@@ -11,14 +9,18 @@ time/space optimizations predicated upon the following assumptions:
     want to do that, you have to create the new crossfilter, using the new
     data, from scratch.
 
-The actual backing store for a dimension is a TypedArray, enabling significant
+In addition, this implementation is easier to use with a "redux" style
+app, as all operations on the crossfilter are immutable (ie, return a
+new crossfilter).
+
+The actual backing store for a dimension is a TypedArray, enabling
 performance improvements over the original crossfilter.
 
 There are also a handful of new methods, primarily to take advantage of the
 performance (eg, crossfilter.fillBySelection)
 
-Helpful documents (this module tries to follow the original API as much
-as is feasable):
+Helpful documents (this module follows similar concepts as the original,
+but deviates from the API):
     https://github.com/square/crossfilter/
     http://square.github.io/crossfilter/
 
@@ -26,731 +28,7 @@ There is also a newer, community supported fork of crossfilter, with a
 more complex API.  In a few cases, elements of that API were incorporated.
     https://github.com/square/crossfilter/
 
+See test cases for some concrete examples.
 */
-// XXX replace
-import { polygonContains } from "d3";
 
-import PositiveIntervals from "./positiveIntervals";
-import BitArray from "./bitArray";
-import {
-  makeSortIndex,
-  lowerBound,
-  lowerBoundIndirect,
-  upperBoundIndirect
-} from "./util";
-
-class NotImplementedError extends Error {
-  constructor(...params) {
-    super(...params);
-
-    // Maintains proper stack trace for where our error was thrown (only available on V8)
-    if (Error.captureStackTrace) {
-      Error.captureStackTrace(this, NotImplementedError);
-    }
-  }
-}
-
-class TypedCrossfilter {
-  constructor(data) {
-    this.data = data;
-
-    // filters: array of { id, dimension }
-    this.filters = [];
-    this.selection = new BitArray(data.length);
-    this.updateTime = 0;
-  }
-
-  size() {
-    return this.data.length;
-  }
-
-  all() {
-    return this.data;
-  }
-
-  /*
-  Create a crossfilter dimension, upon which filtering (subselection) can
-  be done.  Each dimension is typed, and has a particular set of filtering
-  semantics.
-    * ScalarDimension - backed by TypedArray values, supporting filtering
-      by value (within a value range, or one or more exact values)
-    * EnumDimension - backed by an enumeration (eg, strings, bools), filtering
-      by one or more enum categories.
-    * SpatialDimension - backed by 2D points, filter by containment within
-      various shapes (currently supports within Rectangle and within Polygon).
-  Call this method to create a dimension, passing arguments appropriate for
-  the dimension constructor.
-  */
-  dimension(DimensionType, ...rest) {
-    const id = this.selection.allocDimension();
-    const dim = new DimensionType(this, id, ...rest);
-    this.filters.push({ id, dim });
-    dim.filterAll();
-    return dim;
-  }
-
-  _freeDimension(id) {
-    this.selection.freeDimension(id);
-    this.filters = this.filters.filter(f => f._id !== id);
-  }
-
-  // return array of all records that are selected/filtered
-  // by all dimensions.
-  allFiltered() {
-    const { selection } = this;
-    const res = [];
-    for (let i = 0, len = this.data.length; i < len; i += 1) {
-      if (selection.isSelected(i)) {
-        res.push(this.data[i]);
-      }
-    }
-    return res;
-  }
-
-  countFiltered() {
-    return this.selection.selectionCount;
-  }
-
-  isElementFiltered(i) {
-    return this.selection.isSelected(i);
-  }
-
-  // fill array with one of two values, based upon selection state
-  fillByIsFiltered(array, selectedValue, deselectedValue) {
-    return this.selection.fillBySelection(
-      array,
-      selectedValue,
-      deselectedValue
-    );
-  }
-}
-
-// Base dimension type - not exported.
-class _Dimension {
-  constructor(xfltr, id) {
-    this.crossfilter = xfltr;
-    this._id = id;
-    this.groups = [];
-  }
-
-  dispose() {
-    this.crossfilter._freeDimension(this._id);
-    return this;
-  }
-
-  id() {
-    return this._id;
-  }
-
-  _filterUpdate() {
-    this.crossfilter.updateTime += 1;
-  }
-}
-
-// Scalar dimension type - value must be a scalar type (eg, int, float),
-// and value array must be a TypedArray.
-//
-class ScalarDimension extends _Dimension {
-  constructor(xfltr, id, value, ValueArrayType) {
-    super(xfltr, id);
-
-    // current selection filter, expressed as PostiveIntervals.
-    this.currentFilter = [];
-
-    // Two modes - caller can provide a pre-created value array,
-    // or a map function which will create it.
-    let array;
-    if (value instanceof ValueArrayType) {
-      if (value.length !== this.crossfilter.data.length) {
-        throw new RangeError(
-          "ScalarDimension values length must equal crossfilter data record count"
-        );
-      }
-      array = value;
-    } else if (value instanceof Function) {
-      // Create value array
-      array = this._createValueArray(
-        value,
-        new ValueArrayType(this.crossfilter.data.length)
-      );
-    } else {
-      throw new NotImplementedError(
-        "dimension value must be function or value array type"
-      );
-    }
-    this.value = array;
-
-    // create sort index
-    this.index = makeSortIndex(array);
-  }
-
-  _createValueArray(value, array) {
-    // create dimension value array
-    const { data } = this.crossfilter;
-    const len = data.length;
-    const larray = array;
-    for (let i = 0; i < len; i += 1) {
-      larray[i] = value(data[i]);
-    }
-    return larray;
-  }
-
-  // Argument is an array of intervals indicating records newly selected/filtered
-  //
-  _updateFilters(newFilter) {
-    const cNewFilter = PositiveIntervals.canonicalize(newFilter);
-
-    const adds = PositiveIntervals.difference(cNewFilter, this.currentFilter);
-    const dels = PositiveIntervals.difference(this.currentFilter, cNewFilter);
-
-    this.crossfilter.filters.forEach(f =>
-      f.dim.groups.forEach(grp => grp._updateReduceDel(this, dels))
-    );
-
-    dels.forEach(interval =>
-      this.crossfilter.selection.deselectIndirectFromRange(
-        this._id,
-        this.index,
-        interval
-      )
-    );
-
-    adds.forEach(interval =>
-      this.crossfilter.selection.selectIndirectFromRange(
-        this._id,
-        this.index,
-        interval
-      )
-    );
-
-    this.crossfilter.filters.forEach(f =>
-      f.dim.groups.forEach(grp => grp._updateReduceAdd(this, adds))
-    );
-
-    this.currentFilter = cNewFilter;
-    this._filterUpdate();
-  }
-
-  // filter by value - exact match
-  filterExact(value) {
-    const newFilter = [
-      lowerBoundIndirect(this.value, this.index, value, 0, this.value.length),
-      upperBoundIndirect(this.value, this.index, value, 0, this.value.length)
-    ];
-    if (newFilter[0] <= newFilter[1]) {
-      this._updateFilters([newFilter]);
-    } else {
-      this._updateFilters([]);
-    }
-    return this;
-  }
-
-  // filter by a set of values, eg. enum.
-  filterEnum(values) {
-    const newFilter = [];
-    for (let v = 0, len = values.length; v < len; v += 1) {
-      const intv = [
-        lowerBoundIndirect(
-          this.value,
-          this.index,
-          values[v],
-          0,
-          this.value.length
-        ),
-        upperBoundIndirect(
-          this.value,
-          this.index,
-          values[v],
-          0,
-          this.value.length
-        )
-      ];
-      if (intv[0] <= intv[1]) newFilter.push(intv);
-    }
-    this._updateFilters(newFilter);
-    return this;
-  }
-
-  // filter by value range [lo, hi)
-  // lo: inclusive, hi: exclusive
-  filterRange(range) {
-    const newFilter = [];
-    const intv = [
-      lowerBoundIndirect(
-        this.value,
-        this.index,
-        range[0],
-        0,
-        this.value.length
-      ),
-      upperBoundIndirect(this.value, this.index, range[1], 0, this.value.length)
-    ];
-    if (intv[0] < intv[1]) newFilter.push(intv);
-    this._updateFilters(newFilter);
-    return this;
-  }
-
-  // select all - equivalent of selecting all in this dimension
-  filterAll() {
-    this._updateFilters([[0, this.value.length]]);
-    return this;
-  }
-
-  // select none
-  filterNone() {
-    this._updateFilters([]);
-  }
-
-  // return top k records, starting with offset, in descending order.
-  // Order is this dimension's sort order
-  top(k, offset = 0) {
-    const { data, selection } = this.crossfilter;
-    const { index } = this;
-    const len = index.length;
-    const ret = [];
-    let i = 0;
-    let skip = 0;
-    let found = 0;
-
-    // skip up to offset records
-    for (i = len - 1; i >= 0 && skip < offset; i -= 1) {
-      if (selection.isSelected(index[i])) {
-        skip += 1;
-      }
-    }
-
-    // grab up to k records
-    for (; i >= 0 && found < k; i -= 1) {
-      if (selection.isSelected(index[i])) {
-        ret.push(data[index[i]]);
-        found += 1;
-      }
-    }
-
-    return ret;
-  }
-
-  // return bottom k records, starting with offset, in ascending order.
-  // Order is this dimension's sort order
-  bottom(k, offset = 0) {
-    const { data, selection } = this.crossfilter;
-    const { index } = this;
-    const len = index.length;
-    const ret = [];
-    let skip = 0;
-    let found = 0;
-    let i = 0;
-
-    // skip up to offset records
-    for (i = 0; i < len && skip < offset; i += 1) {
-      if (selection.isSelected(index[i])) {
-        skip += 1;
-      }
-    }
-
-    // grab up to k records
-    for (; i < len && found < k; i += 1) {
-      if (selection.isSelected(index[i])) {
-        ret.push(data[index[i]]);
-        found += 1;
-      }
-    }
-
-    return ret;
-  }
-
-  group(groupValue) {
-    const grp = new ScalarGroup(groupValue, this.value.constructor, this);
-    this.groups.push(grp);
-    return grp;
-  }
-
-  _freeGroup(group) {
-    this.groups = this.groups.filter(e => e !== group);
-  }
-}
-
-// Ordered enumeration - supports any sortable enumerable type, eg,
-// strings, which can be mapped into an fixed numeric range [0..n).
-//
-class EnumDimension extends ScalarDimension {
-  constructor(xfltr, id, value) {
-    super(xfltr, id, value, Uint32Array);
-  }
-
-  _createValueArray(value, array) {
-    const { data } = this.crossfilter;
-    const len = data.length;
-    const larray = array;
-
-    // create enumeration table - mapping between the value
-    // and the enum.
-    const s = new Set();
-    for (let i = 0; i < len; i += 1) {
-      s.add(value(data[i]));
-    }
-    this.enumIndex = Array.from(s);
-    this.enumIndex.sort();
-
-    // create dimension value array
-    const enumLen = this.enumIndex.length;
-    for (let i = 0; i < len; i += 1) {
-      const v = value(data[i]);
-      const e = lowerBound(this.enumIndex, v, 0, enumLen);
-      larray[i] = e;
-    }
-    return larray;
-  }
-
-  filterExact(value) {
-    return super.filterExact(
-      lowerBound(this.enumIndex, value, 0, this.enumIndex.length)
-    );
-  }
-
-  filterEnum(values) {
-    return super.filterEnum(
-      values.map(v => lowerBound(this.enumIndex, v, 0, this.enumIndex.length))
-    );
-  }
-
-  filterRange(range) {
-    return super.filterEnum(
-      range.map(v => lowerBound(this.enumIndex, v, 0, this.enumIndex.length))
-    );
-  }
-
-  group(groupValue) {
-    const grp = new EnumGroup(groupValue, this.value.constructor, this);
-    this.groups.push(grp);
-    return grp;
-  }
-}
-
-/*
-Super simple 2D spatial dimension, supporting basic "filter within"
-operations.
-*/
-class SpatialDimension extends _Dimension {
-  constructor(xfltr, id, X, Y) {
-    super(xfltr, id);
-
-    if (X.length !== Y.length && X.length !== this.crossfilter.data.length) {
-      throw new RangeError(
-        "SpatialDimension values must have same dimensionality as crossfilter"
-      );
-    }
-    this.X = X;
-    this.Y = Y;
-
-    this.Xindex = makeSortIndex(X);
-    this.Yindex = makeSortIndex(Y);
-  }
-
-  filterAll() {
-    this.crossfilter.selection.selectAll(this._id);
-    this._filterUpdate();
-  }
-
-  filterNone() {
-    this.crossfilter.selection.deselectAll(this._id);
-    this._filterUpdate();
-  }
-
-  /*
-  this could be smarter, but we don't currently use it...
-  */
-  filterWithinRect(northwest, southeast) {
-    const [x0, y0] = northwest;
-    const [x1, y1] = southeast;
-    const { X, Y } = this;
-    const seln = this.crossfilter.selection;
-    const { _id } = this;
-    seln.deselectAll(_id);
-    for (let i = 0, l = this.X.length; i < l; i += 1) {
-      const x = X[i];
-      const y = Y[i];
-      if (x0 <= x && x < x1 && y0 <= y && y < y1) {
-        seln.selectOne(_id, i);
-      }
-    }
-    this._filterUpdate();
-  }
-
-  /*
-  Relatively brute force filter by polygon.   Polygon is array of points, where
-  each point is [x,y].   Eg, [[x0,y0], [x1,y1], ...].
-
-  Currently uses d3.polygonContains() to test for polygon inclusion, which itself
-  uses a ray casting (crossing number) algorithm.  There are a series of optimizations
-  to make this faster:
-    * first sliced by X or Y, using an index on the axis
-    * then the polygon bounding box is used for trivial rejection
-    * then the polygon test is applied
-  */
-  filterWithinPolygon(polygon) {
-    /* return bounding box of the polygon */
-    function polygonBoundingBox(pg) {
-      let minX = Number.MAX_VALUE;
-      let minY = Number.MAX_VALUE;
-      let maxX = Number.MIN_VALUE;
-      let maxY = Number.MIN_VALUE;
-      for (let i = 0, l = pg.length; i < l; i += 1) {
-        const p = pg[i];
-        const x = p[0];
-        const y = p[1];
-        if (x < minX) minX = x;
-        if (y < minY) minY = y;
-        if (x > maxX) maxX = x;
-        if (y > maxY) maxY = y;
-      }
-      return [minX, minY, maxX, maxY];
-    }
-
-    const [minX, minY, maxX, maxY] = polygonBoundingBox(polygon);
-    const { X, Y } = this;
-    let slice;
-    let index;
-    if (maxY - minY > maxX - minX) {
-      slice = [
-        lowerBoundIndirect(X, this.Xindex, minX, 0, X.length),
-        upperBoundIndirect(X, this.Xindex, maxX, 0, X.length)
-      ];
-      index = this.Xindex;
-    } else {
-      slice = [
-        lowerBoundIndirect(Y, this.Yindex, minY, 0, Y.length),
-        upperBoundIndirect(Y, this.Yindex, maxY, 0, Y.length)
-      ];
-      index = this.Yindex;
-    }
-
-    const seln = this.crossfilter.selection;
-    const { _id } = this;
-    const testWithin = polygonContains; // d3.polygonContains()
-    seln.deselectAll(_id);
-
-    for (let i = slice[0], e = slice[1]; i < e; i += 1) {
-      const rid = index[i];
-      const x = X[rid];
-      const y = Y[rid];
-      if (
-        minX <= x &&
-        x < maxX &&
-        minY <= y &&
-        y < maxY &&
-        testWithin(polygon, [x, y])
-      ) {
-        seln.selectOne(_id, rid);
-      }
-    }
-    this._filterUpdate();
-  }
-}
-
-// Groups!  Map/reduce
-//
-class ScalarGroup {
-  constructor(groupValue, groupValueType, dimension) {
-    // parent dimension
-    this.dimension = dimension;
-
-    // generate group names from dimension values
-    this.mapValue = this.constructor._map(
-      groupValue,
-      groupValueType,
-      dimension
-    );
-
-    // group index is mapping from data record index to group index
-    this.groupIndex = new Uint32Array(dimension.crossfilter.data.length);
-
-    // default to counting
-    this.reduceCount();
-
-    // Creates this.groups
-    this._reduce();
-  }
-
-  // internal support function - map all dimension values to group values.
-  //
-  static _map(groupValue, GroupValueType, dimension) {
-    // groupValue is optional.  Defaults to identity.  Used to perform
-    // initial map operation.
-    //
-    // identity: save some memory...
-    if (groupValue === undefined) return dimension.value;
-
-    const data = dimension.value;
-    const len = data.length;
-    const mapValue = new GroupValueType(dimension.value.length);
-    for (let i = 0; i < len; i += 1) {
-      mapValue[i] = groupValue(data[i]);
-    }
-    return mapValue;
-  }
-
-  // Update the group reduction incrementally.  Called when *any* dimension filter
-  // changes.  Guaranteed to be called AFTER the crossfilter is updated.
-  //
-  // Arguments:
-  //  * dim: the dimension that is changing
-  //  * intv: interval list of newly selected values on `dim` (adds)
-  //
-  _updateReduceAdd(dim, intv) {
-    // ignore updates to self, as we don't reduce inclusive of our filter
-    if (dim === this.dimension || intv.length === 0) return;
-
-    // Each item in the range was just added to `dim`.  It was NOT previously
-    // selected - reduceAdd if it is now selected.
-    const { data, selection } = this.dimension.crossfilter;
-    intv.forEach(rng => {
-      for (let r = rng[0]; r < rng[1]; r += 1) {
-        const i = dim.index[r];
-        if (selection.isSelectedIgnoringDim(i, this.dimension.id())) {
-          const group = this.groups[this.groupIndex[i]];
-          group.value = this.reduceAdd(group.value, data[i]);
-        }
-      }
-    });
-  }
-
-  // Update the group reduction incrementally.  Called when *any* dimension filter
-  // changes.   Guaranteed to be called BEFORE the crossfilter is updated.
-  //
-  // Arguments:
-  //  * dim: the dimension that is changing
-  //  * intv: interval list of previously selected values on `dim` (dels)
-  //
-  _updateReduceDel(dim, intv) {
-    // ignore updates to self, as we don't reduce inclusive of our filter
-    if (dim === this.dimension || intv.length === 0) return;
-
-    // Each item in the range will be remved from `dim`.  reduceRemove if it
-    // is currently selected.
-    const { data, selection } = this.dimension.crossfilter;
-    intv.forEach(rng => {
-      for (let r = rng[0]; r < rng[1]; r += 1) {
-        const i = dim.index[r];
-        if (selection.isSelectedIgnoringDim(i, this.dimension.id())) {
-          const group = this.groups[this.groupIndex[i]];
-          group.value = this.reduceRemove(group.value, data[i]);
-        }
-      }
-    });
-  }
-
-  // Reduce the entire data set, creating both the group index and the
-  // groups data.
-  //
-  _reduce() {
-    const { dimension } = this;
-    const { data } = dimension.crossfilter;
-
-    // Create groups
-    const groupNames = new Set(this.mapValue);
-    this.groups = [];
-    const groupIndexByName = {};
-    groupNames.forEach(name => {
-      this.groups.push({ key: name, value: this.reduceInitial() });
-      groupIndexByName[name] = this.groups.length - 1;
-    });
-
-    // Create groupIndex - index map between data record index and group index
-    for (let i = 0, len = this.mapValue.length; i < len; i += 1) {
-      this.groupIndex[i] = groupIndexByName[this.mapValue[i]];
-    }
-
-    // reduce all filtered records, IGNORING the current dimension's filter
-    const { selection } = dimension.crossfilter;
-    for (let i = 0, len = data.length; i < len; i += 1) {
-      if (selection.isSelectedIgnoringDim(i, dimension.id())) {
-        const group = this.groups[this.groupIndex[i]];
-        group.value = this.reduceAdd(group.value, data[i]);
-      }
-    }
-  }
-
-  dispose() {
-    this.dimension._freeGroup(this);
-    return this;
-  }
-
-  // return number of distinct values in the group, independent of any filters.
-  //
-  size() {
-    return this.groups.length;
-  }
-
-  // Set the reduce functions and return the grouping.
-  //
-  reduce(add, remove, initial) {
-    this.reduceAdd = add;
-    this.reduceRemove = remove;
-    this.reduceInitial = initial;
-    this._reduce();
-    return this;
-  }
-
-  // set the reduce functions to count records.
-  reduceCount() {
-    return this.reduce(p => p + 1, p => p - 1, () => 0);
-  }
-
-  // set the reduce functions to sum records using specified value accessor.
-  //
-  reduceSum(value) {
-    return this.reduce((p, v) => p + value(v), (p, v) => p - value(v), () => 0);
-  }
-
-  all() {
-    const res = [...this.groups];
-    res.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
-    return res;
-  }
-}
-
-class EnumGroup extends ScalarGroup {
-  static _map(groupValue, groupValueType, dimension) {
-    // groupValue is optional.  Defaults to identity.  Used to perform
-    // initial map operation.
-    //
-    // identity: save some memory
-    if (groupValue === undefined) return dimension.value;
-
-    // non-identity mapping unsupported for EnumDimension/EnumGroup.
-    // XXX: this could be implemented, but would require another index
-    // array to map from the group names/keys back to the dimension values.
-    // With this, we just rely on the dimensions `enumIndex` to map from
-    // enumeration value to the record.
-    throw new NotImplementedError("enumerated group mapping not implemented");
-  }
-
-  all() {
-    const res = [];
-    this.groups.forEach(e =>
-      res.push({
-        // XXX: assumes identity group map - see comment in _map()
-        key: this.dimension.enumIndex[e.key],
-        value: e.value
-      })
-    );
-    res.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
-    return res;
-  }
-}
-
-// Wrapper for backwards compat with crossfilter.
-//
-function crossfilter(data) {
-  return new TypedCrossfilter(data);
-}
-
-crossfilter.PositiveIntervals = PositiveIntervals;
-crossfilter.BitArray = BitArray;
-crossfilter.TypedCrossfilter = TypedCrossfilter;
-crossfilter.ScalarDimension = ScalarDimension;
-crossfilter.EnumDimension = EnumDimension;
-crossfilter.SpatialDimension = SpatialDimension;
-
-export default crossfilter;
+export { default } from "./crossfilter";

makefile

@@ -15,6 +15,7 @@ build-server : build-client
 	cp -r server/* $(SERVERBUILD)
 	cp -r client/build/  $(CLIENTBUILD)
 	mkdir -p $(SERVERBUILD)/app/web/static/img
+	mkdir -p $(SERVERBUILD)/app/web/templates/
 	cp $(CLIENTBUILD)/index.html $(SERVERBUILD)/app/web/templates/
 	cp -r $(CLIENTBUILD)/static $(SERVERBUILD)/app/web/
 	cp $(CLIENTBUILD)/favicon.png $(SERVERBUILD)/app/web/static/img
@@ -28,6 +29,8 @@ build-client :
 # If you are actively developing in the server folder use this, dirties the source tree
 build-for-server-dev : clean-server build-client
 	mkdir -p server/app/web/static/img
+	mkdir -p server/app/web/static/js
+	mkdir -p server/app/web/templates/
 	cp client/build/index.html server/app/web/templates/
 	cp -r client/build/static server/app/web/
 	cp client/build/favicon.png server/app/web/static/img
@@ -63,8 +66,10 @@ release-stage-1 : dev-env bump clean-lite gen-package-lock
 # build dist and release to dev pypi
 release-stage-2 : dev-env pydist twine
 	@echo "Dist built and uploaded to test.pypi.org"
-	@echo "Test the install `make install-release-test` and then upload to Pypi prod"
-	@echo "`make twine-prod`"
+	@echo "Test the install:"
+	@echo "    make install-release-test"
+	@echo "Then upload to Pypi prod:"
+	@echo "    make twine-prod"
 
 release-stage-final: twine-prod
 	@echo "Release uploaded to pypi.org"
@@ -73,7 +78,8 @@ release-stage-final: twine-prod
 # use this if you accidently burned a test release version number,
 release-burned : dev-env pydist twine-prod
 	@echo "Dist built and uploaded to pypi.org"
-	@echo "Test the install `make install-release`"
+	@echo "Test the install:"
+	@echo "    make install-release"
 
 dev-env :
 	pip install -r server/requirements-dev.txt
@@ -117,6 +123,6 @@ install-release : uninstall
 	@echo "Installed cellxgene from pypi.org"
 
 uninstall :
-	yes | pip uninstall cellxgene || true
+	pip uninstall -y cellxgene || :
 
 .PHONY : install install-dev install-release-test install-release uninstall

server/app/app.py

@@ -4,7 +4,6 @@ from flask import Flask
 from flask_caching import Cache
 from flask_compress import Compress
 from flask_cors import CORS
-from flask_restful_swagger_2 import get_swagger_blueprint
 
 from .rest_api.rest import get_api_resources
 from .util.utils import Float32JSONEncoder
@@ -26,21 +25,7 @@ app.config.update(SECRET_KEY=SECRET_KEY)
 # Application Data
 data = None
 
-# A list of swagger document objects
-docs = []
 resources = get_api_resources()
-docs.append(resources.get_swagger_doc())
-
 app.register_blueprint(webapp.bp)
 app.register_blueprint(resources.blueprint)
-app.register_blueprint(
-    get_swagger_blueprint(
-        docs,
-        "/api/swagger",
-        produces=["application/json"],
-        title="cellxgene rest api",
-        description="An API connecting ExpressionMatrix2 clustering algorithm to cellxgene",
-    )
-)
-
 app.add_url_rule("/", endpoint="index")

server/app/driver/driver.py

@@ -42,45 +42,12 @@ class CXGDriver(metaclass=ABCMeta):
         pass
 
     @abstractmethod
-    def filter_dataframe(self, filter):
-        """
-        Filter cells from data and return a subset of the data. They can operate on both obs and var dimension with
-        indexing and filtering by annotation value. Filters are combined with the and operator.
-        See REST specs for info on filter format:
-        https://github.com/chanzuckerberg/cellxgene/blob/master/docs/REST_API.md
-
-        :param filter: dictionary with filter params
-        :return: View into scanpy object with cells/genes filtered
-        """
-        pass
-
-    @abstractmethod
-    def annotation(self, filter, axis, fields=None):
+    def annotation_to_fbs_matrix(self, axis, field=None):
         """
         Gets annotation value for each observation
-        :param filter: filter: dictionary with filter params
         :param axis: string obs or var
         :param fields: list of keys for annotation to return, returns all annotation values if not set.
-        :return: dict: names - list of fields in order, data - list of lists or metadata
-        [observation ids, val1, val2...]
-        """
-        pass
-
-    @abstractmethod
-    def annotation_to_fbs_matrix(self, axis, field=None):
-        """ Same as annotation(), except returns a flatbuffer, and does not support filtering. """
-        pass
-
-    @abstractmethod
-    def data_frame(self, filter, axis):
-        """
-        Retrieves data for each variable for observations in data frame
-        :param filter: filter: dictionary with filter params
-        :param axis: string obs or var
-        :return: {
-            "var": list of variable ids,
-            "obs": [cellid, var1 expression, var2 expression, ...],
-        }
+        :return: flatbuffer: in fbs/matrix.fbs encoding
         """
         pass
 
@@ -104,16 +71,6 @@ class CXGDriver(metaclass=ABCMeta):
         """
         pass
 
-    @abstractmethod
-    def layout(self, filter, interactive_limit=None):
-        """
-        Computes a n-d layout for cells through dimensionality reduction.
-        :param filter: filter: dictionary with filter params
-        :param interactive_limit: -- don't compute if total # genes in dataframes are larger than this
-        :return:  [cellid, x, y, ...]
-        """
-        pass
-
     @abstractmethod
     def layout_to_fbs_matrix(self, filter):
         """ same as layout, except returns a flatbuffer """

server/app/rest_api/rest.py

@@ -3,22 +3,17 @@ import pkg_resources
 import warnings
 
 from flask import Blueprint, current_app, jsonify, make_response, request
-from flask_restful_swagger_2 import Api, swagger, Resource
-from werkzeug.datastructures import ImmutableMultiDict
+from flask_restful import Api, Resource
 
 from server.app.util.constants import (
     Axis,
     DiffExpMode,
     JSON_NaN_to_num_warning_msg,
 )
-from server.app.util.filter import parse_filter, QueryStringError
-from server.app.util.models import FilterModel
-from server.app.util.utils import get_mime_type
 from server.app.util.errors import (
     FilterError,
     InteractiveError,
     JSONEncodingValueError,
-    MimeTypeError,
     PrepareError,
 )
 
@@ -31,47 +26,6 @@ Sort order for routes
 
 
 class SchemaAPI(Resource):
-    @swagger.doc(
-        {
-            "summary": "get schema for dataframe and annotations",
-            "tags": ["initialize"],
-            "parameters": [],
-            "responses": {
-                "200": {
-                    "description": "schema",
-                    "examples": {
-                        "application/json": {
-                            "schema": {
-                                "dataframe": {
-                                    "nObs": 383,
-                                    "nVar": 19944,
-                                    "type": "float32",
-                                },
-                                "annotations": {
-                                    "obs": [
-                                        {"name": "name", "type": "string"},
-                                        {"name": "tissue_type", "type": "string"},
-                                        {"name": "num_reads", "type": "int32"},
-                                        {"name": "sample_name", "type": "string"},
-                                        {
-                                            "name": "clusters",
-                                            "type": "categorical",
-                                            "categories": [99, 1, "unknown cluster"],
-                                        },
-                                        {"name": "QScore", "type": "float32"},
-                                    ],
-                                    "var": [
-                                        {"name": "name", "type": "string"},
-                                        {"name": "gene", "type": "string"},
-                                    ],
-                                },
-                            }
-                        }
-                    },
-                }
-            },
-        }
-    )
     def get(self):
         return make_response(
             jsonify({"schema": current_app.data.schema}), HTTPStatus.OK
@@ -79,47 +33,6 @@ class SchemaAPI(Resource):
 
 
 class ConfigAPI(Resource):
-    @swagger.doc(
-        {
-            "summary": "Configuration information to assist in front-end adaptation"
-            " to underlying engine, available functionality, interactive time limits, etc",
-            "tags": ["initialize"],
-            "parameters": [],
-            "responses": {
-                "200": {
-                    "description": "schema",
-                    "examples": {
-                        "application/json": {
-                            "config": {
-                                "features": [
-                                    {
-                                        "method": "POST",
-                                        "path": "/cluster/",
-                                        "available": False,
-                                    },
-                                    {
-                                        "method": "POST",
-                                        "path": "/layout/obs",
-                                        "available": True,
-                                        "interactiveLimit": 10000,
-                                    },
-                                    {
-                                        "method": "POST",
-                                        "path": "/layout/var",
-                                        "available": False,
-                                    },
-                                ],
-                                "displayNames": {
-                                    "engine": "ScanPy version 1.33",
-                                    "dataset": "/home/joe/mouse/blorth.csv",
-                                },
-                            }
-                        }
-                    },
-                }
-            },
-        }
-    )
     def get(self):
         config = {
             "config": {
@@ -152,57 +65,24 @@ class ConfigAPI(Resource):
                 "parameters": {
                     "max_category_items": current_app.data.max_category_items
                 },
+                "library_versions": {
+                    "scanpy": pkg_resources.get_distribution("scanpy").version,
+                    "cellxgene": pkg_resources.get_distribution("cellxgene").version,
+                    "anndata": pkg_resources.get_distribution("cellxgene").version
+                }
             }
         }
         return make_response(jsonify(config), HTTPStatus.OK)
 
 
 class AnnotationsObsAPI(Resource):
-    @swagger.doc(
-        {
-            "summary": "Fetch annotations (metadata) for all observations.",
-            "tags": ["annotations"],
-            "parameters": [
-                {
-                    "in": "query",
-                    "name": "annotation-name",
-                    "type": "string",
-                    "description": "list of 1 or more annotation names",
-                }
-            ],
-            "responses": {
-                "200": {
-                    "description": "annotations",
-                    "examples": {
-                        "application/json": {
-                            "names": ["tissue_type", "sex", "num_reads", "clusters"],
-                            "data": [
-                                [0, "lung", "F", 39844, 99],
-                                [1, "heart", "M", 83, 1],
-                                [49, "spleen", None, 2, "unknown cluster"],
-                            ],
-                        }
-                    },
-                },
-                "400": {
-                    "description": "one or more of the annotation-name identifiers were not associated with an "
-                    "annotation name"
-                },
-            },
-        }
-    )
     def get(self):
         fields = request.args.getlist("annotation-name", None)
         preferred_mimetype = request.accept_mimetypes.best_match(
-            ["application/json", "application/octet-stream"],
-            "application/json"
+            ["application/octet-stream"]
         )
         try:
-            if preferred_mimetype == "application/json":
-                return make_response(
-                    current_app.data.annotation({}, "obs", fields), HTTPStatus.OK, {"Content-Type": "application/json"}
-                )
-            elif preferred_mimetype == "application/octet-stream":
+            if preferred_mimetype == "application/octet-stream":
                 return make_response(current_app.data.annotation_to_fbs_matrix("obs", fields),
                                      HTTPStatus.OK,
                                      {"Content-Type": "application/octet-stream"})
@@ -210,119 +90,18 @@ class AnnotationsObsAPI(Resource):
                 return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
         except KeyError:
             return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
-        except JSONEncodingValueError as e:
-            # JSON encoding failure, usually due to bad data
-            warnings.warn(JSON_NaN_to_num_warning_msg)
-            return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
-        except ValueError as e:
-            return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
-
-    @swagger.doc(
-        {
-            "summary": "Fetch annotations (metadata) for filtered subset of observations.",
-            "tags": ["annotations"],
-            "parameters": [
-                {
-                    "in": "query",
-                    "name": "annotation-name",
-                    "type": "string",
-                    "description": "list of 1 or more annotation names",
-                },
-                {
-                    "name": "filter",
-                    "description": "Complex Filter",
-                    "in": "body",
-                    "schema": FilterModel,
-                },
-            ],
-            "responses": {
-                "200": {
-                    "description": "annotations",
-                    "examples": {
-                        "application/json": {
-                            "names": ["tissue_type", "sex", "num_reads", "clusters"],
-                            "data": [
-                                [0, "lung", "F", 39844, 99],
-                                [1, "heart", "M", 83, 1],
-                                [49, "spleen", None, 2, "unknown cluster"],
-                            ],
-                        }
-                    },
-                },
-                "400": {
-                    "description": "malformed filter or one or more of the annotation-name identifiers were"
-                    "not associated with an annotation name"
-                },
-            },
-        }
-    )
-    def put(self):
-        fields = request.args.getlist("annotation-name", None)
-        try:
-            annotation_response = current_app.data.annotation(
-                request.get_json()["filter"], "obs", fields
-            )
-            return make_response(
-                annotation_response, HTTPStatus.OK, {"Content-Type": "application/json"}
-            )
-        except KeyError:
-            return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
-        except FilterError as e:
-            return make_response(e.message, HTTPStatus.BAD_REQUEST)
-        except JSONEncodingValueError as e:
-            # JSON encoding failure, usually due to bad data
-            warnings.warn(JSON_NaN_to_num_warning_msg)
-            return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
         except ValueError as e:
             return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
 
 
 class AnnotationsVarAPI(Resource):
-    @swagger.doc(
-        {
-            "summary": "Fetch annotations (metadata) for all variables.",
-            "tags": ["annotations"],
-            "parameters": [
-                {
-                    "in": "query",
-                    "name": "annotation-name",
-                    "type": "string",
-                    "description": "list of 1 or more annotation names",
-                }
-            ],
-            "responses": {
-                "200": {
-                    "description": "annotations",
-                    "examples": {
-                        "application/json": {
-                            "names": ["name", "category"],
-                            "data": [
-                                [0, "ATAD3C", 1],
-                                [1, "RER1", None],
-                                [49, "S100B", 6],
-                            ],
-                        }
-                    },
-                },
-                "400": {
-                    "description": "one or more of the annotation-name identifiers were not associated with an"
-                    " annotation name"
-                },
-            },
-        }
-    )
     def get(self):
         fields = request.args.getlist("annotation-name", None)
         preferred_mimetype = request.accept_mimetypes.best_match(
-            ["application/json", "application/octet-stream"],
-            "application/json"
+            ["application/octet-stream"]
         )
         try:
-            if preferred_mimetype == "application/json":
-                return make_response(current_app.data.annotation({}, "var", fields),
-                                     HTTPStatus.OK,
-                                     {"Content-Type": "application/json"})
-            elif preferred_mimetype == "application/octet-stream":
+            if preferred_mimetype == "application/octet-stream":
                 return make_response(current_app.data.annotation_to_fbs_matrix("var", fields),
                                      HTTPStatus.OK,
                                      {"Content-Type": "application/octet-stream"})
@@ -330,317 +109,22 @@ class AnnotationsVarAPI(Resource):
                 return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
         except KeyError:
             return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
-        except JSONEncodingValueError as e:
-            # JSON encoding failure, usually due to bad data
-            warnings.warn(JSON_NaN_to_num_warning_msg)
-            return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
-        except ValueError as e:
-            return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
-
-    @swagger.doc(
-        {
-            "summary": "Fetch annotations (metadata) for filtered subset of variables.",
-            "tags": ["annotations"],
-            "parameters": [
-                {
-                    "in": "query",
-                    "name": "annotation-name",
-                    "type": "string",
-                    "description": "list of 1 or more annotation names",
-                },
-                {
-                    "name": "filter",
-                    "description": "Complex Filter",
-                    "in": "body",
-                    "schema": FilterModel,
-                },
-            ],
-            "responses": {
-                "200": {
-                    "description": "annotations",
-                    "examples": {
-                        "application/json": {
-                            "names": ["name", "category"],
-                            "data": [
-                                [0, "ATAD3C", 1],
-                                [1, "RER1", None],
-                                [49, "S100B", 6],
-                            ],
-                        }
-                    },
-                },
-                "400": {
-                    "description": "malformed filter or one or more of the annotation-name identifiers were"
-                    "not associated with an annotation name"
-                },
-            },
-        }
-    )
-    def put(self):
-        fields = request.args.getlist("annotation-name", None)
-        try:
-            annotation_response = current_app.data.annotation(
-                request.get_json()["filter"], "var", fields
-            )
-            return make_response(
-                annotation_response, HTTPStatus.OK, {"Content-Type": "application/json"}
-            )
-        except KeyError:
-            return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
-        except FilterError:
-            return make_response("Malformed filter", HTTPStatus.BAD_REQUEST)
-        except JSONEncodingValueError as e:
-            # JSON encoding failure, usually due to bad data
-            warnings.warn(JSON_NaN_to_num_warning_msg)
-            return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
-        except ValueError as e:
-            return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
-
-
-class DataObsAPI(Resource):
-    @swagger.doc(
-        {
-            "summary": "Get data (expression values) from the dataframe.",
-            "tags": ["data"],
-            "parameters": [
-                {
-                    "in": "query",
-                    "name": "filter",
-                    "type": "string",
-                    "description": "axis:key:value",
-                },
-                {
-                    "in": "query",
-                    "name": "accept-type",
-                    "type": "string",
-                    "description": "MIME type",
-                },
-            ],
-            "responses": {
-                "200": {
-                    "description": "expression",
-                    "examples": {
-                        "application/json": {
-                            "var": [0, 20000],
-                            "obs": [[1, 39483, 3902, 203, 0, 0, 28]],
-                        }
-                    },
-                },
-                "400": {"description": "Malformed filter"},
-                "406": {"description": "Unacceptable MIME type"},
-            },
-        }
-    )
-    def get(self):
-        accept_type = request.args.get("accept-type", None)
-        # request.args is immutable
-        args = request.args.copy()
-        args.pop("accept-type", None)
-        try:
-            filter_ = parse_filter(
-                ImmutableMultiDict(args), current_app.data.schema["annotations"]
-            )
-        except QueryStringError as e:
-            return make_response(e.message, HTTPStatus.BAD_REQUEST)
-        # TODO support CSV
-        try:
-            # TODO store mime_type when more than one is supported
-            get_mime_type(
-                acceptable_types=["application/json"],
-                query_param=accept_type,
-                header=request.accept_mimetypes,
-            )
-        except MimeTypeError as e:
-            return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
-        try:
-            return make_response(
-                current_app.data.data_frame(filter_, axis=Axis.OBS),
-                HTTPStatus.OK,
-                {"Content-Type": "application/json"},
-            )
-        except FilterError as e:
-            return make_response(e.message, HTTPStatus.BAD_REQUEST)
-        except JSONEncodingValueError as e:
-            # JSON encoding failure, usually due to bad data
-            warnings.warn(JSON_NaN_to_num_warning_msg)
-            return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
-        except ValueError as e:
-            return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
-
-    @swagger.doc(
-        {
-            "summary": "Get data (expression values) from the dataframe.",
-            "tags": ["data"],
-            "parameters": [
-                {
-                    "name": "filter",
-                    "description": "Complex Filter",
-                    "in": "body",
-                    "schema": FilterModel,
-                }
-            ],
-            "responses": {
-                "200": {
-                    "description": "expression",
-                    "examples": {
-                        "application/json": {
-                            "var": [0, 20000],
-                            "obs": [[1, 39483, 3902, 203, 0, 0, 28]],
-                        }
-                    },
-                },
-                "400": {"description": "Malformed filter"},
-                "406": {"description": "Unacceptable MIME type"},
-            },
-        }
-    )
-    def put(self):
-        if not request.accept_mimetypes.best_match(["application/json", "text/csv"]):
-            return make_response(
-                f"Unsupported MIME type '{request.accept_mimetypes}'",
-                HTTPStatus.NOT_ACCEPTABLE,
-            )
-        try:
-            get_mime_type(
-                acceptable_types=["application/json"], header=request.accept_mimetypes
-            )
-        except MimeTypeError as e:
-            return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
-        try:
-            return make_response(
-                (
-                    current_app.data.data_frame(
-                        request.get_json()["filter"], axis=Axis.OBS
-                    )
-                ),
-                HTTPStatus.OK,
-                {"Content-Type": "application/json"},
-            )
-        except FilterError as e:
-            return make_response(e.message, HTTPStatus.BAD_REQUEST)
-        except JSONEncodingValueError as e:
-            # JSON encoding failure, usually due to bad data
-            warnings.warn(JSON_NaN_to_num_warning_msg)
-            return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
         except ValueError as e:
             return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
 
 
 class DataVarAPI(Resource):
-    @swagger.doc(
-        {
-            "summary": "Get data (expression values) from the dataframe.",
-            "tags": ["data"],
-            "parameters": [
-                {
-                    "in": "query",
-                    "name": "filter",
-                    "type": "string",
-                    "description": "axis:key:value",
-                },
-                {
-                    "in": "query",
-                    "name": "accept-type",
-                    "type": "string",
-                    "description": "MIME type",
-                },
-            ],
-            "responses": {
-                "200": {
-                    "description": "expression",
-                    "examples": {
-                        "application/json": {
-                            "obs": [0, 20000],
-                            "var": [[1, 39483, 3902, 203, 0, 0, 28]],
-                        }
-                    },
-                },
-                "400": {"description": "Malformed filter"},
-                "406": {"description": "Unacceptable MIME type"},
-            },
-        }
-    )
-    def get(self):
-        accept_type = request.args.get("accept-type", None)
-        # request.args is immutable
-        args = request.args.copy()
-        args.pop("accept-type", None)
-        try:
-            filter_ = parse_filter(
-                ImmutableMultiDict(args), current_app.data.schema["annotations"]
-            )
-        except QueryStringError as e:
-            return make_response(e.message, HTTPStatus.BAD_REQUEST)
-        try:
-            get_mime_type(
-                acceptable_types=["application/json"],
-                query_param=accept_type,
-                header=request.accept_mimetypes,
-            )
-        except MimeTypeError as e:
-            return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
-        try:
-            return make_response(
-                current_app.data.data_frame(filter_, axis=Axis.VAR),
-                HTTPStatus.OK,
-                {"Content-Type": "application/json"},
-            )
-        except FilterError as e:
-            return make_response(e.message, HTTPStatus.BAD_REQUEST)
-        except JSONEncodingValueError as e:
-            # JSON encoding failure, usually due to bad data
-            warnings.warn(JSON_NaN_to_num_warning_msg)
-            return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
-        except ValueError as e:
-            return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
-
-    @swagger.doc(
-        {
-            "summary": "Get data (expression values) from the dataframe.",
-            "tags": ["data"],
-            "parameters": [
-                {
-                    "name": "filter",
-                    "description": "Complex Filter",
-                    "in": "body",
-                    "schema": FilterModel,
-                }
-            ],
-            "responses": {
-                "200": {
-                    "description": "expression",
-                    "examples": {
-                        "application/json": {
-                            "obs": [0, 20000],
-                            "var": [[1, 39483, 3902, 203, 0, 0, 28]],
-                        }
-                    },
-                },
-                "400": {"description": "Malformed filter"},
-                "406": {"description": "Unacceptable MIME type"},
-            },
-        }
-    )
     def put(self):
         preferred_mimetype = request.accept_mimetypes.best_match(
-            ["application/json", "application/octet-stream"],
-            "application/json"
+            ["application/octet-stream"]
         )
         try:
-            if preferred_mimetype == "application/json":
-                return make_response(
-                    (
-                        current_app.data.data_frame(
-                            request.get_json()["filter"], axis=Axis.VAR
-                        )
-                    ),
-                    HTTPStatus.OK,
-                    {"Content-Type": "application/json"},
-                )
-            elif preferred_mimetype == "application/octet-stream":
+            if preferred_mimetype == "application/octet-stream":
+                filter_json = request.get_json()
+                filter = filter_json["filter"] if filter_json else None
                 return make_response(
                     current_app.data.data_frame_to_fbs_matrix(
-                        request.get_json()["filter"], axis=Axis.VAR
+                        filter, axis=Axis.VAR
                     ),
                     HTTPStatus.OK,
                     {"Content-Type": "application/octet-stream"})
@@ -648,73 +132,11 @@ class DataVarAPI(Resource):
                 return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
         except FilterError as e:
             return make_response(e.message, HTTPStatus.BAD_REQUEST)
-        except JSONEncodingValueError as e:
-            # JSON encoding failure, usually due to bad data
-            warnings.warn(JSON_NaN_to_num_warning_msg)
-            return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
         except ValueError as e:
             return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
 
 
 class DiffExpObsAPI(Resource):
-    @swagger.doc(
-        {
-            "summary": "Generate differential expression (DE) statistics for two specified subsets of data, "
-            "as indicated by the two provided observation complex filters",
-            "tags": ["diffexp"],
-            # TODO sort out params
-            # "parameters": [
-            #     # {
-            #     #     "in": "body",
-            #     #     "name": "mode",
-            #     #     "type": "string",
-            #     #     "required": True,
-            #     #     "description": "topN or varFilter"
-            #     # },
-            #     {
-            #         "in": "query",
-            #         "name": "count",
-            #         "type": "int32",
-            #         "description": "TopN mode: how many vars to return"
-            #     },
-            #     {
-            #         "in": "body",
-            #         "name": "varFilter",
-            #         "schema": FilterModel,
-            #         "description": "varFilter: Complex filter, only var for which vars to return"
-            #     },
-            #     {
-            #         "in": "body",
-            #         "name": "set1",
-            #         "schema": FilterModel,
-            #         "required": True,
-            #         "description": "Complex filter, only obs - observations in set1"
-            #     },
-            #     {
-            #         "in": "body",
-            #         "name": "set2",
-            #         "schema": FilterModel,
-            #         "description": "Complex filter, only obs - observations in set2.
-            # If not included, inverse of set1."
-            #     },
-            # ],
-            "responses": {
-                "200": {
-                    "description": "Statistics are encoded as an array of arrays, with fields ordered as: "
-                    "varIndex, logfoldchange,  pVal, pValAdj",
-                    "examples": {
-                        "application/json": [
-                            [328, -2.569_489, 2.655_706e-63, 3.642_036e-57],
-                            [1250, -2.569_489, 2.655_706e-63, 3.642_036e-57],
-                        ]
-                    },
-                },
-                "400": {"description": "malformed filter"},
-                "403": {"description": "non-interactive request"},
-                "501": {"description": "diffexp is not implemented"},
-            },
-        }
-    )
     def post(self):
         args = request.get_json()
         # confirm mode is present and legal
@@ -783,40 +205,12 @@ class DiffExpObsAPI(Resource):
 
 
 class LayoutObsAPI(Resource):
-    @swagger.doc(
-        {
-            "summary": "Get the default layout for all observations.",
-            "tags": ["layout"],
-            "parameters": [],
-            "responses": {
-                "200": {
-                    "description": "layout",
-                    "examples": {
-                        "application/json": {
-                            "layout": {
-                                "ndims": 2,
-                                "coordinates": [
-                                    [0, 0.284_483, 0.983_744],
-                                    [1, 0.038_844, 0.739_444],
-                                ],
-                            }
-                        }
-                    },
-                },
-                "400": {"description": "Data preparation error"},
-            },
-        }
-    )
     def get(self):
         preferred_mimetype = request.accept_mimetypes.best_match(
-            ["application/json", "application/octet-stream"],
-            "application/json"
+            ["application/octet-stream"]
         )
         try:
-            if preferred_mimetype == "application/json":
-                return make_response(current_app.data.layout({}), HTTPStatus.OK, {"Content-Type": "application/json"})
-
-            elif preferred_mimetype == "application/octet-stream":
+            if preferred_mimetype == "application/octet-stream":
                 return make_response(current_app.data.layout_to_fbs_matrix(),
                                      HTTPStatus.OK,
                                      {"Content-Type": "application/octet-stream"})
@@ -824,69 +218,19 @@ class LayoutObsAPI(Resource):
                 return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
         except PrepareError as e:
             return make_response(e.message, HTTPStatus.INTERNAL_SERVER_ERROR)
-        except JSONEncodingValueError as e:
-            # JSON encoding failure, usually due to bad data
-            warnings.warn(JSON_NaN_to_num_warning_msg)
-            return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
         except ValueError as e:
             return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
 
-    # @swagger.doc({
-    #     "summary": "Observation layout for filtered subset.",
-    #     "tags": ["layout"],
-    #     "parameters": [
-    #         {
-    #             "name": "filter",
-    #             "description": "Complex Filter",
-    #             "in": "body",
-    #             "schema": FilterModel
-    #         }
-    #     ],
-    #     "responses": {
-    #         "200": {
-    #             "description": "layout",
-    #             "examples": {
-    #                 "application/json": {
-    #                     "layout": {
-    #                         "ndims": 2,
-    #                         "coordinates": [
-    #                             [0, 0.284483, 0.983744],
-    #                             [1, 0.038844, 0.739444]
-    #                         ]
-    #                     }
-    #                 }
-    #             }
-    #         },
-    #         "400": {
-    #             "description": "Malformed filter"
-    #         },
-    #         "403": {
-    #             "description": "Non-interactive request"
-    #         },
-    #     }
-    # })
-    # def put(self):
-    #     try:
-    #         filter = request.get_json()["filter"]
-    #         interactive_limit = current_app.data.features["layout"]["obs"]["interactiveLimit"]
-    #         layout = current_app.data.layout(filter, interactive_limit=interactive_limit)
-    #         return make_response(layout, HTTPStatus.OK, {"Content-Type": content_type})
-    #     except FilterError as e:
-    #         return make_response(e.message, HTTPStatus.BAD_REQUEST)
-    #     except InteractiveError:
-    #         return make_response("Non-interactive request", HTTPStatus.FORBIDDEN)
-
 
 def get_api_resources():
     bp = Blueprint("api", __name__, url_prefix="/api/v0.2")
-    api = Api(bp, add_api_spec_resource=False)
+    api = Api(bp)
     # Initialization routes
     api.add_resource(SchemaAPI, "/schema")
     api.add_resource(ConfigAPI, "/config")
     # Data routes
     api.add_resource(AnnotationsObsAPI, "/annotations/obs")
     api.add_resource(AnnotationsVarAPI, "/annotations/var")
-    api.add_resource(DataObsAPI, "/data/obs")
     api.add_resource(DataVarAPI, "/data/var")
     # Computation routes
     api.add_resource(DiffExpObsAPI, "/diffexp/obs")

server/app/scanpy_engine/scanpy_engine.py

@@ -1,16 +1,13 @@
 import warnings
 
 import numpy as np
-from pandas import DataFrame
 from pandas.core.dtypes.dtypes import CategoricalDtype
 import scanpy.api as sc
-from scipy import sparse
 
 from server.app.driver.driver import CXGDriver
 from server.app.util.constants import Axis, DEFAULT_TOP_N
 from server.app.util.errors import (
     FilterError,
-    InteractiveError,
     JSONEncodingValueError,
     PrepareError,
     ScanpyFileError,
@@ -146,6 +143,9 @@ class ScanpyEngine(CXGDriver):
                 "using `cellxgene prepare`, please run `cellxgene prepare --help` for more "
                 "information."
             )
+        except MemoryError:
+            raise ScanpyFileError("Error while loading file: out of memory, file is too large"
+                                  " for memory available")
         except Exception as e:
             raise ScanpyFileError(
                 f"Error while loading file: {e}, File must be in the .h5ad format, please check "
@@ -197,23 +197,6 @@ class ScanpyEngine(CXGDriver):
                 f"to solve this problem. "
             )
 
-    def filter_dataframe(self, filter):
-        """
-         Filter cells from data and return a subset of the data. They can operate on both obs and var dimension with
-         indexing and filtering by annotation value. Filters are combined with the and operator.
-         See REST specs for info on filter format:
-         # TODO update this link to swagger when it's done
-         https://docs.google.com/document/d/1Fxjp1SKtCk7l8QP9-7KAjGXL0eldi_qEnNT0NmlGzXI/edit#heading=h.8qc9q57amldx
-
-        :param filter: dictionary with filter params
-        :return: View into scanpy object with cells/genes filtered
-        """
-        if not filter:
-            return self.data
-        obs_selector, var_selector = self._filter_to_mask(filter, use_slices=False)
-        data = self._slice(self.data, obs_selector, var_selector)
-        return data
-
     @staticmethod
     def _annotation_filter_to_mask(filter, d_axis, count):
         mask = np.ones((count,), dtype=bool)
@@ -277,72 +260,6 @@ class ScanpyEngine(CXGDriver):
                 )
         return obs_selector, var_selector
 
-    @staticmethod
-    def _slice(data, obs_selector=None, vars_selector=None):
-        """
-        Slice date using any selector that the AnnData object
-        supprots for slicing.  If selector is None, will not slice
-        on that axis.
-
-        This method exists to optimize filtering/slicing sparse data that has
-        access patterns which impact slicing performance.
-
-        https://docs.scipy.org/doc/scipy/reference/sparse.html
-        """
-        prefer_row_access = (
-            sparse.isspmatrix_csr(data._X)
-            or sparse.isspmatrix_lil(data._X)
-            or sparse.isspmatrix_bsr(data._X)
-        )
-        if prefer_row_access:
-            # Row-major slicing
-            if obs_selector is not None:
-                data = data[obs_selector, :]
-            if vars_selector is not None:
-                data = data[:, vars_selector]
-        else:
-            # Col-major slicing
-            if vars_selector is not None:
-                data = data[:, vars_selector]
-            if obs_selector is not None:
-                data = data[obs_selector, :]
-
-        return data
-
-    def annotation(self, filter, axis, fields=None):
-        """
-         Gets annotation value for each observation
-        :param filter: filter: dictionary with filter params
-        :param axis: string obs or var
-        :param fields: list of keys for annotation to return, returns all annotation values if not set.
-        :return: dict: names - list of fields in order, data - list of lists or metadata
-        [observation ids, val1, val2...]
-        """
-        try:
-            obs_selector, var_selector = self._filter_to_mask(filter)
-        except (KeyError, IndexError) as e:
-            raise FilterError(f"Error parsing filter: {e}") from e
-        if axis == Axis.OBS:
-            obs = self.data.obs[obs_selector]
-            if not fields:
-                fields = obs.columns.tolist()
-            result = {
-                "names": fields,
-                "data": DataFrame(obs[fields]).to_records(index=True).tolist(),
-            }
-        else:
-            var = self.data.var[var_selector]
-            if not fields:
-                fields = var.columns.tolist()
-            result = {
-                "names": fields,
-                "data": DataFrame(var[fields]).to_records(index=True).tolist(),
-            }
-        try:
-            return jsonify_scanpy(result)
-        except ValueError:
-            raise JSONEncodingValueError("Error encoding annotations to JSON")
-
     def annotation_to_fbs_matrix(self, axis, fields=None):
         if axis == Axis.OBS:
             df = self.data.obs
@@ -352,44 +269,6 @@ class ScanpyEngine(CXGDriver):
             df = df[fields]
         return encode_matrix_fbs(df, col_idx=df.columns)
 
-    def data_frame(self, filter, axis):
-        """
-        Retrieves data for each variable for observations in data frame
-        :param filter: filter: dictionary with filter params
-        :param axis: string obs or var
-        :return: {
-            "var": list of variable ids,
-            "obs": [cellid, var1 expression, var2 expression, ...],
-        }
-        """
-        try:
-            obs_selector, var_selector = self._filter_to_mask(filter)
-        except (KeyError, IndexError) as e:
-            raise FilterError(f"Error parsing filter: {e}") from e
-        _X = self.data._X[obs_selector, var_selector]
-        if sparse.issparse(_X):
-            _X = _X.toarray()
-        var_index_sliced = self.data.var.index[var_selector]
-        obs_index_sliced = self.data.obs.index[obs_selector]
-        if axis == Axis.OBS:
-            result = {
-                "var": var_index_sliced.tolist(),
-                "obs": DataFrame(_X, index=obs_index_sliced)
-                .to_records(index=True)
-                .tolist(),
-            }
-        else:
-            result = {
-                "obs": obs_index_sliced.tolist(),
-                "var": DataFrame(_X.T, index=var_index_sliced)
-                .to_records(index=True)
-                .tolist(),
-            }
-        try:
-            return jsonify_scanpy(result)
-        except ValueError:
-            raise JSONEncodingValueError("Error encoding dataframe to JSON")
-
     def data_frame_to_fbs_matrix(self, filter, axis):
         """
         Retrieves data 'X' and returns in a flatbuffer Matrix.
@@ -405,7 +284,7 @@ class ScanpyEngine(CXGDriver):
             raise ValueError("Only VAR dimension access is supported")
         try:
             obs_selector, var_selector = self._filter_to_mask(filter, use_slices=False)
-        except (KeyError, IndexError) as e:
+        except (KeyError, IndexError, TypeError) as e:
             raise FilterError(f"Error parsing filter: {e}") from e
         if obs_selector is not None:
             raise FilterError("filtering on obs unsupported")
@@ -440,50 +319,6 @@ class ScanpyEngine(CXGDriver):
                 "Error encoding differential expression to JSON"
             )
 
-    def layout(self, filter, interactive_limit=None):
-        """
-        Computes a n-d layout for cells through dimensionality reduction.
-        :param filter: filter: dictionary with filter params
-        :param interactive_limit: -- don't compute if total # genes in dataframes are larger than this
-        :return:  [cellid, x, y, ...]
-        """
-        try:
-            df = self.filter_dataframe(filter)
-        except (KeyError, IndexError) as e:
-            raise FilterError(f"Error parsing filter: {e}") from e
-        if interactive_limit and len(df.obs.index) > interactive_limit:
-            raise InteractiveError("Size data is too large for interactive computation")
-        # TODO Filtering cells is fine, but filtering genes does nothing because the neighbors are
-        # calculated using the original vars (geneset) and this doesn’t get updated when you use less.
-        # Need to recalculate neighbors (long) if user requests new layout filtered by var
-        # TODO for MVP we are pushing computation of layout to preprocessing and not allowing re-layout
-        # this will probably change after user feedback
-        # getattr(sc.tl, self.layout_method)(df, random_state=123)
-        try:
-            df_layout = df.obsm[f"X_{self.layout_method}"]
-        except ValueError as e:
-            raise PrepareError(
-                f"Layout has not been calculated using {self.layout_method}, "
-                f"please prepare your datafile and relaunch cellxgene"
-            ) from e
-        normalized_layout = DataFrame(
-            (df_layout - df_layout.min()) / (df_layout.max() - df_layout.min()),
-            index=df.obs.index,
-        )
-        try:
-            return jsonify_scanpy(
-                {
-                    "layout": {
-                        "ndims": normalized_layout.shape[1],
-                        "coordinates": normalized_layout.to_records(
-                            index=True
-                        ).tolist(),
-                    }
-                }
-            )
-        except ValueError:
-            raise JSONEncodingValueError("Error encoding layout to JSON")
-
     def layout_to_fbs_matrix(self):
         """
         Return the default 2-D layout for cells as a FBS Matrix.
@@ -493,10 +328,15 @@ class ScanpyEngine(CXGDriver):
         * only returns Matrix in columnar layout
         """
         try:
-            df_layout = self.data.obsm[f"X_{self.layout_method}"]
+            full_embedding = self.data.obsm[f"X_{self.layout_method}"]
+            if full_embedding.shape[1] > 2:
+                warnings.warn(f"Warning: found {full_embedding.shape[1]} \
+                components of embedding. Using the first two for layout display.")
+            df_layout = full_embedding[:, :2]
         except ValueError as e:
             raise PrepareError(
                 f"Layout has not been calculated using {self.layout_method}, "
                 f"please prepare your datafile and relaunch cellxgene") from e
+
         normalized_layout = (df_layout - df_layout.min()) / (df_layout.max() - df_layout.min())
         return encode_matrix_fbs(normalized_layout.astype(dtype=np.float32), col_idx=None, row_idx=None)

server/app/util/filter.py

@@ -1,86 +0,0 @@
-import json
-from collections import defaultdict
-
-from numpy import float32, int32
-
-from server.app.util.constants import Axis
-
-
-class QueryStringError(Exception):
-    def __init__(self, key, message):
-        self.key = key
-        self.message = message
-
-
-def _convert_variable(datatype, variable):
-    """
-    Convert variable to number (float/int)
-    Used for dataset metadata and for query string
-    :param datatype: type to convert to
-    :param variable (string or None): value of variable
-    :return: converted variable
-    :raises: AssertionError
-    """
-    assert datatype in ["boolean", "categorical", "float32", "int32", "string"]
-    if variable is None:
-        return variable
-    if datatype == "int32":
-        variable = int32(variable)
-    elif datatype == "float32":
-        variable = float32(variable)
-    elif datatype == "boolean":
-        variable = json.loads(variable)
-        assert isinstance(variable, bool)
-    return variable
-
-
-def parse_filter(query_filter, schema):
-    """
-    The filter comes in as arguments from a GET request
-    For categorical metadata keys filter based on axis:key=value
-    For continuous metadata keys filter by axis:key=min,max
-    Either value can be replaced by a * To have only a minimum
-    value axis:key=min,* To have only a maximum value axis:key=*,max
-
-    They combine via AND so a cell's metadata would have to match every filter
-
-    The results is a matrix with the cells the pass the filter and at this point all the genes
-    :param query_filter: flask's request.args
-    :param schema: dictionary schema
-    :raises QueryStringError
-    :return:
-    """
-    query = defaultdict(lambda: defaultdict(list))
-
-    for key in query_filter:
-        axis, annotation = key.split(":", 1)
-        try:
-            Axis(axis)
-        except ValueError:
-            raise QueryStringError(key, f"Error: key {key} not in metadata schema")
-        ann_filter = {"name": annotation}
-        for ann in schema[axis]:
-            if ann["name"] == annotation:
-                dtype = ann["type"]
-                break
-        else:
-            raise QueryStringError(key, f"Error: {annotation} not a valid annotation name")
-        if dtype in ["string", "categorical", "boolean"]:
-            ann_filter["values"] = [_convert_variable(dtype, i) for i in query_filter.getlist(key)]
-        else:
-            value = query_filter.get(key)
-            try:
-                min_, max_ = value.split(",")
-            except ValueError:
-                raise QueryStringError(key, f"Error: min,max format required for range for {annotation}, got {value}")
-            if min_ == "*":
-                min_ = None
-            if max_ == "*":
-                max_ = None
-            try:
-                ann_filter["min"] = _convert_variable(dtype, min_)
-                ann_filter["max"] = _convert_variable(dtype, max_)
-            except ValueError:
-                raise QueryStringError(key, f"Error: expected type {query[key]['type']} for key {key}, got {value}")
-        query[axis]["annotation_value"].append(ann_filter)
-    return query

server/app/util/models.py

@@ -1,34 +0,0 @@
-from flask_restful_swagger_2 import Schema
-
-
-class AnnotationModel(Schema):
-    type = "object"
-    description = "Filter by annotation key: value"
-    properties = {
-        "name": {"type": "string"},
-        # TODO update to OpenAPI v3.0 when a library is available that supports it
-        # Unfortunately 2.0 doesn't have a way to have a schema that accepts multiple types
-        # Overloading the type key with a list seems to work ok and makes it to the page
-        "values": {"type": "array", "items": {"type": ["float32", "string", "int32", "bool"]}},
-        "min": {"type": ["int32", "float32"]},
-        "max": {"type": ["int32", "float32"]},
-    }
-    required = ["name"]
-
-
-class IndexModel(Schema):
-    type = "object"
-    description = "Filter by index of observation/variable ex. [0, 5, 15]"
-    properties = {"index": {"type": "array", "items": {"format": "int32", "type": "integer"}}}
-
-
-class AxisModel(Schema):
-    type = "object"
-    description = "Axis of data -- obs or var"
-    properties = {"index": IndexModel, "annotation_value": AnnotationModel.array()}
-
-
-class FilterModel(Schema):
-    type = "object"
-    description = "Complex filter"
-    properties = {"filter": {"type": "object", "properties": {"obs": AxisModel, "var": AxisModel}}}

server/app/util/utils.py

@@ -1,10 +1,6 @@
 import json
-from argparse import ArgumentTypeError
-
 from numpy import float32, integer
 
-from server.app.util.errors import MimeTypeError
-
 
 class Float32JSONEncoder(json.JSONEncoder):
     def __init__(self, *args, **kwargs):
@@ -30,31 +26,5 @@ def custom_format_warning(msg, *args, **kwargs):
     return f"[cellxgene] Warning: {msg} \n"
 
 
-def get_mime_type(
-    default="application/json", acceptable_types=["application/json", "text/csv"], query_param=None, header=None
-):
-    mime_type = default
-    if query_param:
-        if query_param in acceptable_types:
-            mime_type = query_param
-        else:
-            raise MimeTypeError(f"Unsupported mime type {query_param} specified in query parameter 'accept-type'")
-    elif len(header):
-        mime_type = header.best_match(acceptable_types)
-        if not mime_type:
-            raise MimeTypeError(f"Unsupported mime type(s) {header} in HTTP Accept header")
-    return mime_type
-
-
-def whole_number(value):
-    try:
-        value = int(value)
-    except ValueError as e:
-        raise ArgumentTypeError(f"{value} is not type int") from e
-    if value < 0:
-        raise ArgumentTypeError(f"{value} is not >= 0")
-    return value
-
-
 def jsonify_scanpy(data):
     return json.dumps(data, cls=Float32JSONEncoder, allow_nan=False)

server/app/web/templates/swagger.html

@@ -1,97 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-	<meta charset="UTF-8">
-	<title>cellxgene REST API - Swagger definition</title>
-	<link href="https://fonts.googleapis.com/css?family=Open+Sans:400,700|Source+Code+Pro:300,600|Titillium+Web:400,600,700"
-		  rel="stylesheet">
-	<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.2.1/swagger-ui.css"
-		  crossorigin="anonymous"/>
-	<style>
-		html {
-			box-sizing: border-box;
-			overflow: -moz-scrollbars-vertical;
-			overflow-y: scroll;
-		}
-
-		*,
-		*:before,
-		*:after {
-			box-sizing: inherit;
-		}
-
-		body {
-			margin: 0;
-			background: #fafafa;
-		}
-	</style>
-</head>
-
-<body>
-
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"
-	 style="position:absolute;width:0;height:0">
-	<defs>
-		<symbol viewBox="0 0 20 20" id="unlocked">
-			<path d="M15.8 8H14V5.6C14 2.703 12.665 1 10 1 7.334 1 6 2.703 6 5.6V6h2v-.801C8 3.754 8.797 3 10 3c1.203 0 2 .754 2 2.199V8H4c-.553 0-1 .646-1 1.199V17c0 .549.428 1.139.951 1.307l1.197.387C5.672 18.861 6.55 19 7.1 19h5.8c.549 0 1.428-.139 1.951-.307l1.196-.387c.524-.167.953-.757.953-1.306V9.199C17 8.646 16.352 8 15.8 8z"></path>
-		</symbol>
-
-		<symbol viewBox="0 0 20 20" id="locked">
-			<path d="M15.8 8H14V5.6C14 2.703 12.665 1 10 1 7.334 1 6 2.703 6 5.6V8H4c-.553 0-1 .646-1 1.199V17c0 .549.428 1.139.951 1.307l1.197.387C5.672 18.861 6.55 19 7.1 19h5.8c.549 0 1.428-.139 1.951-.307l1.196-.387c.524-.167.953-.757.953-1.306V9.199C17 8.646 16.352 8 15.8 8zM12 8H8V5.199C8 3.754 8.797 3 10 3c1.203 0 2 .754 2 2.199V8z"/>
-		</symbol>
-
-		<symbol viewBox="0 0 20 20" id="close">
-			<path d="M14.348 14.849c-.469.469-1.229.469-1.697 0L10 11.819l-2.651 3.029c-.469.469-1.229.469-1.697 0-.469-.469-.469-1.229 0-1.697l2.758-3.15-2.759-3.152c-.469-.469-.469-1.228 0-1.697.469-.469 1.228-.469 1.697 0L10 8.183l2.651-3.031c.469-.469 1.228-.469 1.697 0 .469.469.469 1.229 0 1.697l-2.758 3.152 2.758 3.15c.469.469.469 1.229 0 1.698z"/>
-		</symbol>
-
-		<symbol viewBox="0 0 20 20" id="large-arrow">
-			<path d="M13.25 10L6.109 2.58c-.268-.27-.268-.707 0-.979.268-.27.701-.27.969 0l7.83 7.908c.268.271.268.709 0 .979l-7.83 7.908c-.268.271-.701.27-.969 0-.268-.269-.268-.707 0-.979L13.25 10z"/>
-		</symbol>
-
-		<symbol viewBox="0 0 20 20" id="large-arrow-down">
-			<path d="M17.418 6.109c.272-.268.709-.268.979 0s.271.701 0 .969l-7.908 7.83c-.27.268-.707.268-.979 0l-7.908-7.83c-.27-.268-.27-.701 0-.969.271-.268.709-.268.979 0L10 13.25l7.418-7.141z"/>
-		</symbol>
-
-
-		<symbol viewBox="0 0 24 24" id="jump-to">
-			<path d="M19 7v4H5.83l3.58-3.59L8 6l-6 6 6 6 1.41-1.41L5.83 13H21V7z"/>
-		</symbol>
-
-		<symbol viewBox="0 0 24 24" id="expand">
-			<path d="M10 18h4v-2h-4v2zM3 6v2h18V6H3zm3 7h12v-2H6v2z"/>
-		</symbol>
-
-	</defs>
-</svg>
-
-<div id="swagger-ui"></div>
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.2.1/swagger-ui-bundle.js"
-		 crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.2.1/swagger-ui-standalone-preset.js"
-		crossorigin="anonymous"></script>
-<script>
-	window.onload = function () {
-		const ui = SwaggerUIBundle({
-			url: window.location.href.replace(/\/swagger$/, "") + "/api/swagger.json",
-			dom_id: '#swagger-ui',
-			deepLinking: true,
-			presets: [
-				SwaggerUIBundle.presets.apis,
-				SwaggerUIStandalonePreset
-			],
-			plugins: [
-				SwaggerUIBundle.plugins.DownloadUrl
-			],
-			layout: "StandaloneLayout"
-		});
-
-		window.ui = ui
-	}
-</script>
-</body>
-
-</html>
-
-
-

server/app/web/webapp.py

@@ -11,13 +11,6 @@ def index():
     return render_template("index.html", datasetTitle=dataset_title)
 
 
-# renders swagger documentation
-@bp.route("/swagger")
-def swag():
-    return render_template("swagger.html")
-
-
-# renders swagger documentation
 @bp.route("/favicon.png")
 def favicon():
     return send_from_directory(os.path.join(bp.root_path, "static/img/"), "favicon.png")

server/cli/cli.py

@@ -5,7 +5,7 @@ from .prepare import prepare
 
 
 @click.group(name="cellxgene", context_settings=dict(max_content_width=85))
-@click.version_option(version="0.6.1", prog_name="cellxgene", message="[%(prog)s] Version %(version)s")
+@click.version_option(version="0.8.0", prog_name="cellxgene", message="[%(prog)s] Version %(version)s")
 def cli():
     pass
 

server/cli/launch.py

@@ -14,7 +14,12 @@ from server.app.util.utils import custom_format_warning
 @click.command()
 @click.argument("data", metavar="<data file>", type=click.Path(exists=True, file_okay=True, dir_okay=False))
 @click.option(
-    "--layout", "-l", type=click.Choice(["umap", "tsne"]), default="umap", show_default=True, help="Method for layout."
+    "--layout",
+    "-l",
+    type=click.Choice(["umap", "tsne", "draw_graph_fa", "draw_graph_fr", "diffmap", "phate"]),
+    default="umap",
+    show_default=True,
+    help="Method for layout."
 )
 @click.option(
     "--diffexp",

server/requirements.txt

@@ -5,11 +5,11 @@ Flask-Caching>=1.4.0
 Flask-Compress>=1.4.0
 Flask-Cors>=3.0.6
 Flask-RESTful>=0.3.6
-flask-restful-swagger-2>=0.35
 flatbuffers>=1.10.0
 matplotlib>=2.2
-numpy>=1.14.5
+numpy>=1.15.2
 pandas>=0.23.1
 scanpy>=1.3.2
 scipy>=1.1.0
 scikit-learn>=0.19.1,!=0.20.0
+tables>=3.5.1

server/test/test_api.py

@@ -54,19 +54,10 @@ class EndPoints(unittest.TestCase):
         self.assertEqual(result.status_code, HTTPStatus.OK)
         self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data = result.json()
+        self.assertIn("library_versions", result_data["config"])
         self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k")
         self.assertEqual(len(result_data["config"]["features"]), 4)
 
-    def test_get_layout(self):
-        endpoint = "layout/obs"
-        url = f"{URL_BASE}{endpoint}"
-        result = self.session.get(url)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data = result.json()
-        self.assertEqual(result_data["layout"]["ndims"], 2)
-        self.assertEqual(len(result_data["layout"]["coordinates"]), 2638)
-
     def test_get_layout_fbs(self):
         endpoint = "layout/obs"
         url = f"{URL_BASE}{endpoint}"
@@ -82,53 +73,11 @@ class EndPoints(unittest.TestCase):
         self.assertIsNone(df['row_idx'])
         self.assertEqual(len(df['columns']), df['n_cols'])
 
-    # def test_put_layout(self):
-    #     endpoint = "layout/obs"
-    #     url = f"{URL_BASE}{endpoint}"
-    #     obs_filter = {
-    #         "filter": {
-    #             "obs": {
-    #                 "annotation_value": [
-    #                     {"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
-    #                     {"name": "n_counts", "min": 3000},
-    #                 ],
-    #                 "index": [1, 99, [1000, 2000]]
-    #             }
-    #         }
-    #     }
-    #     result = self.session.put(url, json=obs_filter)
-    #     self.assertEqual(result.status_code, HTTPStatus.OK)
-    #     result_data = result.json()
-    #     self.assertEqual(len(result_data["layout"]["coordinates"]), 15)
-
     def test_bad_filter(self):
-        endpoints = ["annotations/obs", "annotations/var", "data/obs", "data/var"]
-        for endpoint in endpoints:
-            url = f"{URL_BASE}{endpoint}"
-            result = self.session.put(url, json=BAD_FILTER)
-            self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
-
-    def test_get_annotations_obs(self):
-        endpoint = "annotations/obs"
+        endpoint = "data/var"
         url = f"{URL_BASE}{endpoint}"
-        result = self.session.get(url)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data = result.json()
-        self.assertEqual(result_data["names"], ["name", "n_genes", "percent_mito", "n_counts", "louvain"])
-        self.assertEqual(len(result_data["data"]), 2638)
-        self.assertEqual(len(result_data["data"][0]), 6)
-
-    def test_get_annotations_obs_keys(self):
-        endpoint = "annotations/obs"
-        query = "annotation-name=n_genes&annotation-name=percent_mito"
-        url = f"{URL_BASE}{endpoint}?{query}"
-        result = self.session.get(url)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        result_data = result.json()
-        self.assertEqual(result_data["names"], ["n_genes", "percent_mito"])
-        self.assertEqual(len(result_data["data"][0]), 3)
+        result = self.session.put(url, json=BAD_FILTER)
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
 
     def test_get_annotations_obs_fbs(self):
         endpoint = "annotations/obs"
@@ -146,6 +95,23 @@ class EndPoints(unittest.TestCase):
         self.assertEqual(len(df['columns']), df['n_cols'])
         self.assertListEqual(df['col_idx'], ['name', 'n_genes', 'percent_mito', 'n_counts', 'louvain'])
 
+    def test_get_annotations_obs_keys_fbs(self):
+        endpoint = "annotations/obs"
+        query = "annotation-name=n_genes&annotation-name=percent_mito"
+        url = f"{URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df['n_rows'], 2638)
+        self.assertEqual(df['n_cols'], 2)
+        self.assertIsNotNone(df['columns'])
+        self.assertIsNotNone(df['col_idx'])
+        self.assertIsNone(df['row_idx'])
+        self.assertEqual(len(df['columns']), df['n_cols'])
+        self.assertListEqual(df['col_idx'], ['n_genes', 'percent_mito'])
+
     def test_get_annotations_obs_error(self):
         endpoint = "annotations/obs"
         query = "annotation-name=notakey"
@@ -153,50 +119,6 @@ class EndPoints(unittest.TestCase):
         result = self.session.get(url)
         self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
 
-    def test_put_annotations_obs(self):
-        endpoint = "annotations/obs"
-        url = f"{URL_BASE}{endpoint}"
-        obs_filter = {
-            "filter": {
-                "obs": {
-                    "annotation_value": [
-                        {"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
-                        {"name": "n_counts", "min": 3000},
-                    ],
-                    "index": [1, 99, [1000, 2000]],
-                }
-            }
-        }
-        result = self.session.put(url, json=obs_filter)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data = result.json()
-        self.assertEqual(result_data["names"], ["name", "n_genes", "percent_mito", "n_counts", "louvain"])
-        self.assertEqual(len(result_data["data"]), 15)
-
-    def test_filter_put_annotations_obs(self):
-        endpoint = "annotations/obs"
-        query = "annotation-name=n_genes&annotation-name=percent_mito"
-        url = f"{URL_BASE}{endpoint}?{query}"
-        obs_filter = {
-            "filter": {
-                "obs": {
-                    "annotation_value": [
-                        {"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
-                        {"name": "n_counts", "min": 3000},
-                    ],
-                    "index": [1, 99, [1000, 2000]],
-                }
-            }
-        }
-        result = self.session.put(url, json=obs_filter)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data = result.json()
-        self.assertEqual(result_data["names"], ["n_genes", "percent_mito"])
-        self.assertEqual(len(result_data["data"][0]), 3)
-        self.assertEqual(len(result_data["data"]), 15)
-
     def test_diff_exp(self):
         endpoint = "diffexp/obs"
         url = f"{URL_BASE}{endpoint}"
@@ -227,28 +149,6 @@ class EndPoints(unittest.TestCase):
         result_data = result.json()
         self.assertEqual(len(result_data), 10)
 
-    def test_get_annotations_var(self):
-        endpoint = "annotations/var"
-        url = f"{URL_BASE}{endpoint}"
-        result = self.session.get(url)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data = result.json()
-        self.assertEqual(result_data["names"], ["name", "n_cells"])
-        self.assertEqual(len(result_data["data"]), 1838)
-        self.assertEqual(len(result_data["data"][0]), 3)
-
-    def test_get_annotations_var_keys(self):
-        endpoint = "annotations/var"
-        query = "annotation-name=n_cells"
-        url = f"{URL_BASE}{endpoint}?{query}"
-        result = self.session.get(url)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data = result.json()
-        self.assertEqual(result_data["names"], ["n_cells"])
-        self.assertEqual(len(result_data["data"][0]), 2)
-
     def test_get_annotations_var_fbs(self):
         endpoint = "annotations/var"
         url = f"{URL_BASE}{endpoint}"
@@ -265,6 +165,23 @@ class EndPoints(unittest.TestCase):
         self.assertEqual(len(df['columns']), df['n_cols'])
         self.assertListEqual(df['col_idx'], ['name', 'n_cells'])
 
+    def test_get_annotations_var_keys_fbs(self):
+        endpoint = "annotations/var"
+        query = "annotation-name=n_cells"
+        url = f"{URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df['n_rows'], 1838)
+        self.assertEqual(df['n_cols'], 1)
+        self.assertIsNotNone(df['columns'])
+        self.assertIsNotNone(df['col_idx'])
+        self.assertIsNone(df['row_idx'])
+        self.assertEqual(len(df['columns']), df['n_cols'])
+        self.assertListEqual(df['col_idx'], ['n_cells'])
+
     def test_get_annotations_var_error(self):
         endpoint = "annotations/var"
         query = "annotation-name=notakey"
@@ -272,95 +189,36 @@ class EndPoints(unittest.TestCase):
         result = self.session.get(url)
         self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
 
-    def test_put_annotations_var(self):
-        endpoint = "annotations/var"
-        url = f"{URL_BASE}{endpoint}"
-        var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}]}}}
-        result = self.session.put(url, json=var_filter)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data = result.json()
-        self.assertEqual(result_data["names"], ["name", "n_cells"])
-        self.assertEqual(len(result_data["data"]), 2)
-
-    def test_filter_put_annotations_var(self):
-        endpoint = "annotations/var"
-        query = "annotation-name=n_cells"
-        url = f"{URL_BASE}{endpoint}?{query}"
-        var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}]}}}
-        result = self.session.put(url, json=var_filter)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data = result.json()
-        self.assertEqual(result_data["names"], ["n_cells"])
-        self.assertEqual(len(result_data["data"][0]), 2)
-        self.assertEqual(len(result_data["data"]), 2)
-
-    def test_get_data(self):
-        for axis in ["obs", "var"]:
-            endpoint = f"data/{axis}"
-            query = "accept-type=application/json"
-            url = f"{URL_BASE}{endpoint}?{query}"
-            result = self.session.get(url)
-            self.assertEqual(result.status_code, HTTPStatus.OK)
-            self.assertEqual(result.headers["Content-Type"], "application/json")
-            result_data = result.json()
-            self.assertEqual(len(result_data["obs"]), 2638)
-
     def test_data_mimetype_error(self):
-        for axis in ["obs", "var"]:
-            endpoint = f"data/{axis}"
-            query = "accept-type=xxx"
-            url = f"{URL_BASE}{endpoint}?{query}"
-            result = self.session.get(url)
-            self.assertEqual(result.status_code, HTTPStatus.NOT_ACCEPTABLE)
-            url = f"{URL_BASE}{endpoint}"
-            header = {"Accept": "sdkljfa;dsjalkj"}
-            result = self.session.get(url, headers=header)
-            self.assertEqual(result.status_code, HTTPStatus.NOT_ACCEPTABLE)
+        endpoint = f"data/var"
+        header = {"Accept": "xxx"}
+        url = f"{URL_BASE}{endpoint}"
+        result = self.session.put(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.NOT_ACCEPTABLE)
 
-    def test_json_default(self):
-        for axis in ["obs", "var"]:
-            endpoint = f"data/{axis}"
-            url = f"{URL_BASE}{endpoint}"
-            result = self.session.get(url)
-            self.assertEqual(result.status_code, HTTPStatus.OK)
-            self.assertEqual(result.headers["Content-Type"], "application/json")
-
-    def test_data_filter(self):
-        for axis in ["obs", "var"]:
-            endpoint = f"data/{axis}"
-            query = "accept-type=application/json&obs:louvain=NK cells&obs:louvain=CD8 T cells&obs:n_counts=3000,*"
-            url = f"{URL_BASE}{endpoint}?{query}"
-            result = self.session.get(url)
-            self.assertEqual(result.status_code, HTTPStatus.OK)
-            self.assertEqual(result.headers["Content-Type"], "application/json")
-            result_data = result.json()
-            self.assertEqual(len(result_data["obs"]), 38)
-
-    def test_data_json_put(self):
-        for axis in ["obs", "var"]:
-            endpoint = f"data/{axis}"
-            url = f"{URL_BASE}{endpoint}"
-            header = {"Accept": "application/json"}
-            obs_filter = {
-                "filter": {
-                    "obs": {
-                        "annotation_value": [
-                            {"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
-                            {"name": "n_counts", "min": 3000},
-                        ],
-                        "index": [1, 99, [1000, 2000]],
-                    }
-                }
-            }
-            result = self.session.put(url, headers=header, json=obs_filter)
-            self.assertEqual(result.status_code, HTTPStatus.OK)
-            self.assertEqual(result.headers["Content-Type"], "application/json")
-            result_data = result.json()
-            self.assertEqual(len(result_data["obs"]), 15)
+    def test_fbs_default(self):
+        endpoint = f"data/var"
+        url = f"{URL_BASE}{endpoint}"
+        result = self.session.put(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
 
     def test_data_put_fbs(self):
+        endpoint = f"data/var"
+        url = f"{URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.put(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df['n_rows'], 2638)
+        self.assertEqual(df['n_cols'], 1838)
+        self.assertIsNotNone(df['columns'])
+        self.assertListEqual(df['col_idx'].tolist(), [])
+        self.assertIsNone(df['row_idx'])
+        self.assertEqual(len(df['columns']), df['n_cols'])
+
+    def test_data_put_filter_fbs(self):
         endpoint = f"data/var"
         url = f"{URL_BASE}{endpoint}"
         header = {"Accept": "application/octet-stream"}
@@ -384,94 +242,16 @@ class EndPoints(unittest.TestCase):
         self.assertListEqual(df['col_idx'].tolist(), [0, 1, 4])
 
     def test_data_put_single_var(self):
-        for axis in ["obs", "var"]:
-            endpoint = f"data/{axis}"
-            url = f"{URL_BASE}{endpoint}"
-            header = {"Accept": "application/json"}
-            var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}}}
-            result = self.session.put(url, headers=header, json=var_filter)
-            self.assertEqual(result.status_code, HTTPStatus.OK)
-            self.assertEqual(result.headers["Content-Type"], "application/json")
-            result_data = result.json()
-            if axis == "obs":
-                self.assertEqual(len(result_data["obs"][0]), 2)
-                self.assertEqual(len(result_data["var"]), 1)
-            elif axis == "var":
-                self.assertEqual(len(result_data["obs"]), 2638)
-                self.assertEqual(len(result_data["var"][0]), 2639)
-
-    def test_cache(self):
-        endpoint = "annotations/var"
+        endpoint = f"data/var"
         url = f"{URL_BASE}{endpoint}"
-        f1 = {
-            "filter": {
-                "var": {
-                    "annotation_value": [
-                        {
-                            "name": "name",
-                            "values": [
-                                "HLA-DRB1",
-                                "HLA-DQA1",
-                                "HLA-DQB1",
-                                "HLA-DPA1",
-                                "HLA-DPB1",
-                                "MS4A1",
-                                "IL32",
-                                "CCL5",
-                                "CD79B",
-                                "CD79A",
-                            ],
-                        }
-                    ]
-                }
-            }
-        }
-        result = self.session.put(url, json=f1)
+        header = {"Accept": "application/octet-stream"}
+        var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}}}
+        result = self.session.put(url, headers=header, json=var_filter)
         self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data1 = result.json()
-        f2 = {
-            "filter": {
-                "var": {
-                    "annotation_value": [
-                        {
-                            "name": "name",
-                            "values": ["FGFBP2", "GZMA", "LTB", "PRF1", "CTSW", "GZMH", "CCL5", "CCL4", "CST7", "NKG7"],
-                        }
-                    ]
-                }
-            }
-        }
-        result = self.session.put(url, json=f2)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data2 = result.json()
-        self.assertNotEqual(result_data1, result_data2)
-
-    def test_cache_nofilter(self):
-        endpoint = "annotations/var"
-        url = f"{URL_BASE}{endpoint}"
-        f1 = {"filter": {}}
-        result = self.session.put(url, json=f1)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        result_data1 = result.json()
-        f2 = {
-            "filter": {
-                "var": {
-                    "annotation_value": [
-                        {
-                            "name": "name",
-                            "values": ["FGFBP2", "GZMA", "LTB", "PRF1", "CTSW", "GZMH", "CCL5", "CCL4", "CST7", "NKG7"],
-                        }
-                    ]
-                }
-            }
-        }
-        result = self.session.put(url, json=f2)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data2 = result.json()
-        self.assertNotEqual(result_data1, result_data2)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
 
     def test_static(self):
         endpoint = "static"

server/test/test_filter.py

@@ -1,82 +0,0 @@
-import json
-from os import path
-import unittest
-
-from numpy import float32, int32
-from werkzeug.datastructures import ImmutableMultiDict
-
-from server.app.util.filter import _convert_variable, parse_filter, QueryStringError
-
-
-class UtilTest(unittest.TestCase):
-    """Test Case for endpoints"""
-
-    def setUp(self):
-        with open(path.join(path.dirname(__file__), "schema.json")) as fh:
-            schema = json.load(fh)
-        self.schema = schema["annotations"]
-
-    def test_convert(self):
-        five = _convert_variable("int32", "5")
-        self.assertEqual(five, int32(5))
-
-    def test_convert_zero(self):
-        zero = _convert_variable("int32", "0")
-        self.assertEqual(zero, 0)
-
-    def test_convert_float(self):
-        str_to_convert = "4.38719237129"
-        val = _convert_variable("float32", str_to_convert)
-        self.assertAlmostEqual(val, float32(str_to_convert))
-
-    def test_convert_bool(self):
-        str_to_convert = "false"
-        val = _convert_variable("boolean", str_to_convert)
-        self.assertFalse(val)
-        str_to_convert = "true"
-        val = _convert_variable("boolean", str_to_convert)
-        self.assertTrue(val)
-        str_to_convert = "0"
-        with self.assertRaises(AssertionError):
-            val = _convert_variable("boolean", str_to_convert)
-
-    def test_empty_convert(self):
-        empty = _convert_variable("int32", None)
-        self.assertIsNone(empty)
-
-    def test_bad_convert(self):
-        with self.assertRaises(ValueError):
-            _convert_variable("int32", "5.5")
-
-    def test_bad_datatype(self):
-        with self.assertRaises(AssertionError):
-            _convert_variable("jkasdslkja", 1)
-
-    def test_complex_filter(self):
-        filter_dict = ImmutableMultiDict(
-            [("obs:louvain", "NK cells"), ("obs:louvain", "CD8 T cells"), ("obs:n_counts", "3000,*")]
-        )
-        filter_ = parse_filter(filter_dict, self.schema)
-        self.assertIn("obs", filter_)
-        self.assertEqual(
-            filter_["obs"]["annotation_value"],
-            [
-                {"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
-                {"name": "n_counts", "max": None, "min": 3000.0},
-            ],
-        )
-
-    def test_bad_filter(self):
-        bad_annotation_type = ImmutableMultiDict([("obs:tissue", "lung")])
-        with self.assertRaises(QueryStringError):
-            parse_filter(bad_annotation_type, self.schema)
-        bad_axis = ImmutableMultiDict([("xyz:n_genes", "100,1000")])
-        with self.assertRaises(QueryStringError):
-            parse_filter(bad_axis, self.schema)
-
-    def test_boolean_filter(self):
-        schema = {"obs": [{"name": "bool_filter", "type": "boolean"}]}
-        filter_dict = ImmutableMultiDict([("obs:bool_filter", "false")])
-        filter_ = parse_filter(filter_dict, schema)
-        self.assertIn("obs", filter_)
-        self.assertEqual(filter_["obs"]["annotation_value"], [{"name": "bool_filter", "values": [False]}])

server/test/test_nan_rest.py

@@ -2,6 +2,9 @@ from http import HTTPStatus
 from subprocess import Popen
 import unittest
 import time
+import math
+
+import decode_fbs
 
 import requests
 
@@ -43,9 +46,29 @@ class WithNaNs(unittest.TestCase):
         result = self.session.get(url)
         self.assertEqual(result.status_code, HTTPStatus.OK)
 
-    def test_errors(self):
-        endpoints = ["annotations/obs", "annotations/var", "data/obs", "data/var"]
-        for endpoint in endpoints:
-            url = f"{URL_BASE}{endpoint}"
-            result = self.session.get(url)
-            self.assertEqual(result.status_code, HTTPStatus.INTERNAL_SERVER_ERROR)
+    def test_data(self):
+        endpoint = "data/var"
+        url = f"{URL_BASE}{endpoint}"
+        result = self.session.put(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertTrue(math.isnan(df["columns"][3][3]))
+
+    def test_annotation_obs(self):
+        endpoint = "annotations/obs"
+        url = f"{URL_BASE}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertTrue(math.isnan(df["columns"][2][0]))
+
+    def test_annotation_var(self):
+        endpoint = "annotations/var"
+        url = f"{URL_BASE}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertTrue(math.isnan(df["columns"][2][0]))

server/test/test_nan_scanpy_engine.py

@@ -1,4 +1,3 @@
-import json
 import pytest
 import unittest
 import warnings
@@ -7,7 +6,7 @@ import math
 import decode_fbs
 
 from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
-from server.app.util.errors import JSONEncodingValueError
+from server.app.util.errors import FilterError
 
 
 class NaNTest(unittest.TestCase):
@@ -42,10 +41,15 @@ class NaNTest(unittest.TestCase):
         self.assertEqual(data_frame_var["n_cols"], 100)
         self.assertTrue(math.isnan(data_frame_var["columns"][3][3]))
 
-        with pytest.raises(JSONEncodingValueError):
-            json.loads(self.data.data_frame(None, "obs"))
-        with pytest.raises(JSONEncodingValueError):
-            json.loads(self.data.data_frame(None, "var"))
+        with pytest.raises(FilterError):
+            self.data.data_frame_to_fbs_matrix("an erroneous filter", "var")
+        with pytest.raises(FilterError):
+            filter_ = {
+                "filter": {
+                    "obs": {"index": [1, 99, [200, 300]]}
+                }
+            }
+            self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
 
     def test_dataframe_obs_not_implemented(self):
         with self.assertRaises(ValueError) as cm:
@@ -65,8 +69,3 @@ class NaNTest(unittest.TestCase):
         self.assertEqual(annotations["col_idx"], ["name", "n_cells", "var_with_nans"])
         self.assertEqual(annotations["n_rows"], 100)
         self.assertTrue(math.isnan(annotations["columns"][2][0]))
-
-        with pytest.raises(JSONEncodingValueError):
-            json.loads(self.data.annotation(None, "obs"))
-        with pytest.raises(JSONEncodingValueError):
-            json.loads(self.data.annotation(None, "var"))

server/test/test_scanpy_engine.py

@@ -3,11 +3,13 @@ from os import path
 import pytest
 import time
 import unittest
+import decode_fbs
 
 import numpy as np
 from pandas import Series
 
 from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
+from server.app.util.errors import FilterError
 
 
 class UtilTest(unittest.TestCase):
@@ -45,55 +47,29 @@ class UtilTest(unittest.TestCase):
     def test_filter_idx(self):
         filter_ = {
             "filter": {
-                "var": {"index": [1, 99, [200, 300]]},
-                "obs": {"index": [1, 99, [1000, 2000]]},
+                "var": {"index": [1, 99, [200, 300]]}
             }
         }
-        data = self.data.filter_dataframe(filter_["filter"])
-        self.assertEqual(data.shape, (1002, 102))
-
-    def test_filter_annotation(self):
-        filter_ = {
-            "filter": {
-                "obs": {
-                    "annotation_value": [
-                        {"name": "louvain", "values": ["NK cells", "CD8 T cells"]}
-                    ]
-                }
-            }
-        }
-        data = self.data.filter_dataframe(filter_["filter"])
-        self.assertEqual(data.shape, (470, 1838))
-        filter_ = {
-            "filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}
-        }
-        data = self.data.filter_dataframe(filter_["filter"])
-        self.assertEqual(data.shape, (497, 1838))
-
-    def test_filter_annotation_no_uns(self):
-        filter_ = {
-            "filter": {
-                "var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}
-            }
-        }
-        data = self.data.filter_dataframe(filter_["filter"])
-        self.assertEqual(data.shape[1], 1)
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 102)
 
     def test_filter_complex(self):
         filter_ = {
             "filter": {
-                "var": {"index": [1, 99, [200, 300]]},
-                "obs": {
+                "var": {
                     "annotation_value": [
-                        {"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
-                        {"name": "n_counts", "min": 3000},
+                        {"name": "n_cells", "min": 10}
                     ],
-                    "index": [1, 99, [1000, 2000]],
-                },
+                    "index": [1, 99, [200, 300]]
+                }
             }
         }
-        data = self.data.filter_dataframe(filter_["filter"])
-        self.assertEqual(data.shape, (15, 102))
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 91)
 
     def test_obs_and_var_names(self):
         self.assertEqual(np.sum(self.data.data.var["name"].isna()), 0)
@@ -119,60 +95,42 @@ class UtilTest(unittest.TestCase):
         )
 
     def test_layout(self):
-        layout = json.loads(self.data.layout(None))
-        self.assertEqual(layout["layout"]["ndims"], 2)
-        self.assertEqual(len(layout["layout"]["coordinates"]), 2638)
-        self.assertEqual(layout["layout"]["coordinates"][0][0], 0)
-        for idx, val in enumerate(layout["layout"]["coordinates"]):
-            self.assertLessEqual(val[1], 1)
-            self.assertLessEqual(val[2], 1)
+        fbs = self.data.layout_to_fbs_matrix()
+        layout = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(layout["n_cols"], 2)
+        self.assertEqual(layout["n_rows"], 2638)
+
+        X = layout["columns"][0]
+        self.assertTrue((X >= 0).all() and (X <= 1).all())
+        Y = layout["columns"][1]
+        self.assertTrue((Y >= 0).all() and (Y <= 1).all())
 
     def test_annotations(self):
-        annotations = json.loads(self.data.annotation(None, "obs"))
+        fbs = self.data.annotation_to_fbs_matrix("obs")
+        annotations = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(annotations["n_rows"], 2638)
+        self.assertEqual(annotations["n_cols"], 5)
         self.assertEqual(
-            annotations["names"],
+            annotations["col_idx"],
             ["name", "n_genes", "percent_mito", "n_counts", "louvain"],
         )
-        self.assertEqual(len(annotations["data"]), 2638)
-        annotations = json.loads(self.data.annotation(None, "var"))
-        self.assertEqual(annotations["names"], ["name", "n_cells"])
-        self.assertEqual(len(annotations["data"]), 1838)
+
+        fbs = self.data.annotation_to_fbs_matrix("var")
+        annotations = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(annotations['n_rows'], 1838)
+        self.assertEqual(annotations['n_cols'], 2)
+        self.assertEqual(annotations["col_idx"], ["name", "n_cells"])
 
     def test_annotation_fields(self):
-        annotations = json.loads(
-            self.data.annotation(None, "obs", ["n_genes", "n_counts"])
-        )
-        self.assertEqual(annotations["names"], ["n_genes", "n_counts"])
-        self.assertEqual(len(annotations["data"]), 2638)
-        annotations = json.loads(self.data.annotation(None, "var", ["name"]))
-        self.assertEqual(annotations["names"], ["name"])
-        self.assertEqual(len(annotations["data"]), 1838)
+        fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"])
+        annotations = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(annotations["n_rows"], 2638)
+        self.assertEqual(annotations['n_cols'], 2)
 
-    def test_filtered_annotation(self):
-        filter_ = {
-            "filter": {
-                "obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]},
-                "var": {
-                    "annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}]
-                },
-            }
-        }
-        annotations = json.loads(self.data.annotation(filter_["filter"], "obs"))
-        self.assertEqual(
-            annotations["names"],
-            ["name", "n_genes", "percent_mito", "n_counts", "louvain"],
-        )
-        self.assertEqual(len(annotations["data"]), 497)
-        annotations = json.loads(self.data.annotation(filter_["filter"], "var"))
-        self.assertEqual(annotations["names"], ["name", "n_cells"])
-        self.assertEqual(len(annotations["data"]), 2)
-
-    def test_filtered_layout(self):
-        filter_ = {
-            "filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}
-        }
-        layout = json.loads(self.data.layout(filter_["filter"]))
-        self.assertEqual(len(layout["layout"]["coordinates"]), 497)
+        fbs = self.data.annotation_to_fbs_matrix("var", ["name"])
+        annotations = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(annotations['n_rows'], 1838)
+        self.assertEqual(annotations['n_cols'], 1)
 
     def test_diffexp_topN(self):
         f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
@@ -183,42 +141,51 @@ class UtilTest(unittest.TestCase):
         self.assertEqual(len(result), 20)
 
     def test_data_frame(self):
-        data_frame_obs = json.loads(self.data.data_frame(None, "obs"))
-        self.assertEqual(len(data_frame_obs["var"]), 1838)
-        self.assertEqual(len(data_frame_obs["obs"]), 2638)
-        data_frame_var = json.loads(self.data.data_frame(None, "var"))
-        self.assertEqual(len(data_frame_var["var"]), 1838)
-        self.assertEqual(len(data_frame_var["obs"]), 2638)
+        fbs = self.data.data_frame_to_fbs_matrix(None, "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 1838)
+
+        with self.assertRaises(ValueError):
+            self.data.data_frame_to_fbs_matrix(None, "obs")
 
     def test_filtered_data_frame(self):
+        filter_ = {
+            "filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 100}]}}
+        }
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 1040)
+
         filter_ = {
             "filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}
         }
-        data_frame_obs = json.loads(self.data.data_frame(filter_["filter"], "obs"))
-        self.assertEqual(len(data_frame_obs["var"]), 1838)
-        self.assertEqual(len(data_frame_obs["obs"]), 497)
-        self.assertIsInstance(data_frame_obs["obs"][0], (list, tuple))
-        self.assertEqual(type(data_frame_obs["var"][0]), int)
-        data_frame_var = json.loads(self.data.data_frame(filter_["filter"], "var"))
-        self.assertEqual(len(data_frame_var["var"]), 1838)
-        self.assertEqual(len(data_frame_var["obs"]), 497)
-        self.assertIsInstance(data_frame_var["var"][0], (list, tuple))
-        self.assertEqual(type(data_frame_var["obs"][0]), int)
+        with self.assertRaises(FilterError):
+            self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
 
-    def test_data_single_gene(self):
-        for axis in ["obs", "var"]:
-            filter_ = {
-                "filter": {
-                    "var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}
-                }
+    def test_data_named_gene(self):
+        filter_ = {
+            "filter": {
+                "var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}
             }
-            data_frame_var = json.loads(self.data.data_frame(filter_["filter"], axis))
-            if axis == "obs":
-                self.assertEqual(type(data_frame_var["var"][0]), int)
-                self.assertIsInstance(data_frame_var["obs"][0], (list, tuple))
-            elif axis == "var":
-                self.assertEqual(type(data_frame_var["obs"][0]), int)
-                self.assertIsInstance(data_frame_var["var"][0], (list, tuple))
+        }
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 1)
+        self.assertEqual(data["col_idx"], [4])
+
+        filter_ = {
+            "filter": {
+                "var": {"annotation_value": [{"name": "name", "values": ["SPEN", "TYMP", "PRMT2"]}]}
+            }
+        }
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 3)
+        self.assertTrue((data["col_idx"] == [15, 1818, 1837]).all())
 
     if __name__ == "__main__":
         unittest.main()

setup.py

@@ -1,13 +1,4 @@
 from setuptools import setup, find_packages
-import sys
-
-if sys.version_info[0:2] != (3, 6):
-    raise ImportError(
-        "cellxgene currently only supports python 3.6. Python 3.7 is known to fail; we will look at supporting "
-        "versions other than 3.6 in the future."
-        "See https://github.com/chanzuckerberg/cellxgene#conda-and-virtual-environments "
-        "for more help with installation."
-    )
 
 with open("README.md", "rb") as fh:
     long_description = fh.read().decode()
@@ -17,7 +8,7 @@ with open("server/requirements.txt") as fh:
 
 setup(
     name="cellxgene",
-    version="0.6.1",
+    version="0.8.0",
     packages=find_packages(),
     url="https://github.com/chanzuckerberg/cellxgene",
     license="MIT",
@@ -40,6 +31,7 @@ setup(
         "Programming Language :: JavaScript",
         "Programming Language :: Python :: 3",
         "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3 :: Only",
         "Topic :: Scientific/Engineering :: Bio-Informatics",
     ],

travis-build.sh

@@ -0,0 +1,5 @@
+set -eo pipefail
+flake8 server
+npm run --prefix client/ build
+npm run --prefix client/ unit-test
+pytest -s server/test