bumpversion to 0.7.0 (#623)

* initial dataframe commit

* initial dataframe port of core app

* rename variables for clarity

* remove unused import

* comment out unused code

* fix array handling bug in crossfilter dimension creation

* allow creation of empty dataframes

* handle non-existent columns

* handle non-existent columns

* revise tests for new dataframe

* comments for clarity

* comments for clarity

* generate bulk add placeholder with real gene names

* fix bug in gene name adding

* more dataframe unit tests

* fix bug - subset from current world, not universe

* put cut and pasted code into a single function

* improve caching of crossfilter

* remove cascading update bug from graph

* more performance work

* improve state handling for scatterplot

* performance optimization of critical path

* add column summarization

* dataframe utils

* add callOnceLazy

* fix tests

* minor updates found during review

* fix misspelling

* remove RESTv02 from function names

* comment cleanup

* cut/icut col parameter defaults to null

* break up large test

* improve tests and comments on dataframe at/has functions

* add Dataframe withCol/dropCol

* expression varData now stored in a dataframe

* dead code cleanup

* use dataframe.summarize()

* test cases for Dataframe.col.summarize

* update test cases for new dataframe summarize

* improve naming

* use new hasCol API

* add comments

* add more Dataframe.withCol tests

* add ability to specify row index in cut operation

* retire subsetVarData function

* correctly handle expression subsetting

* lint and improve comments

* rename cut to subset

* create helper file for controls reducer

.bumpversion.cfg

@@ -0,0 +1,15 @@
+[bumpversion]
+current_version = 0.7.0
+
+[bumpversion:file:setup.py]
+search = version="{current_version}"
+replace = version="{new_version}"
+
+[bumpversion:file:server/cli/cli.py]
+search = version="{current_version}"
+replace = version="{new_version}"
+
+[bumpversion:file:client/package.json]
+search = "version": "{current_version}"
+replace = "version": "{new_version}"
+

.dockerignore

@@ -0,0 +1,5 @@
+bin
+client
+dist
+docs
+server

.gitignore

@@ -1,16 +1,41 @@
 # dependencies
-node_modules
+node_modules/
 
 # coverage
 coverage
 .nyc_output
 
+.pytest_cache/
+
 # production
-build
+build/
+dist/
+*.egg-info
+
+# Environments
+venv/
+cellxgene/
+
+# client build
+server/app/web/static/css/
+server/app/web/static/img/
+server/app/web/static/media/
+server/app/web/static/js/
+server/app/web/templates/index\.html
+
+# Jupyter Notebook
+.ipynb_checkpoints
+*.ipynb
 
 # misc
 .DS_Store
 npm-debug.log
 .vscode
-
+*.idea*
+__pycache__
+*.DS_Store*
 data
+
+# Jekyll
+docs/_site/
+docs/Gemfile.lock

.travis.yml

@@ -0,0 +1,29 @@
+language: python
+python:
+  - "3.6"
+node_js:
+  - "8"
+cache:
+  pip: true
+install:
+  - set -eo pipefail
+  - pip install flake8
+  - make build
+  - make install
+  - pip install -r server/requirements-dev.txt
+  - docker build .
+
+jobs:
+  include:
+  - name: "Branch Tests"
+    script:
+      - set -eo pipefail
+      - flake8 server
+      - black --check
+      - npm run --prefix client/ build
+      - npm run --prefix client/ unit-test
+      - pytest -s server/test
+  - name: "Smoke Tests"
+    if: branch = master AND type = cron
+    script:
+      - npm run --prefix client/ smoke-test

Dockerfile

@@ -0,0 +1,10 @@
+FROM ubuntu:bionic
+
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+
+RUN apt-get update && \
+    apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev && \
+    pip3 install cellxgene
+
+ENTRYPOINT ["cellxgene"]

MANIFEST.in

@@ -0,0 +1,4 @@
+recursive-include server/app/web/templates *
+recursive-include server/app/web/static *
+
+include server/requirements.txt

README.md

@@ -1,9 +1,288 @@
 # cellxgene
 
-A React + Redux web application for exploring large scale single cell RNA sequence data.
+> an interactive explorer for single-cell transcriptomics data
 
-##### Quickstart:
+`cellxgene` is an interactive data explorer for single-cell transcriptomics datasets, such as those coming from the [Human Cell Atlas](https://humancellatlas.org). Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data, and to demonstrate general, scalable, and reusable patterns for scientific data visualization.
 
-* `npm install`
+<img src="https://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/docs/cellxgene-demo-1.gif" width="200" height="200" hspace="30"><img src="https://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/docs/cellxgene-demo-2.gif" width="200" height="200" hspace="30"><img src="https://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/docs/cellxgene-demo-3.gif" width="200" height="200" hspace="30">
-* `npm start`
+
-* `localhost:3000`
+## getting started
+
+You'll need **python 3.6** and **Google Chrome**. (_Warning_: Python 3.7 is **not** supported at this time)
+The web UI is tested on OSX and Windows using Chrome, and the python CLI is tested on OSX and Ubuntu (via WSL/Windows). It should work on other platforms, but if you run into trouble let us know (see [help](#help-and-contact) below).
+
+To install run
+
+```
+pip install cellxgene
+```
+
+To start exploring a dataset call
+
+```
+cellxgene launch dataset.h5ad --open
+```
+
+If you want an example dataset download [this file](https://github.com/chanzuckerberg/cellxgene/raw/master/example-dataset/pbmc3k.h5ad) and then call
+
+```
+cellxgene launch pbmc3k.h5ad --open
+```
+
+You should see your web browser open with the following
+
+<img width="450" src="https://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/docs/cellxgene-opening-screenshot.png" pad="50px">
+
+**Note**: automatic opening of the browser with the `--open` flag only works on OS X, on other platforms you'll need to directly point to the provided link in your browser.
+
+There are several options available, such as:
+
+- `--layout` to specify the layout as `tsne` or `umap`
+- `--title` to show a title on the explorer
+- `--open` to automatically open the web browser after launching (OS X only)
+
+To see all options call
+
+```
+cellxgene launch --help
+```
+
+There is an additional subcommand called `cellxgene prepare` that takes an existing dataset in one of several formats and applies minimal preprocessing and reformatting so that `launch` can use it (see [the next section](##data-formatting) for more info on `prepare`).
+
+## data formatting
+
+### assumptions
+
+The `launch` command assumes that the data is stored in the `.h5ad` format from the [`anndata`](https://anndata.readthedocs.io/en/latest/index.html) library. It also assumes that certain computations have already been performed. Briefly, the `.h5ad` format wraps a two-dimensional `ndarray` and stores additional metadata as "annotations" for either observations (referred to as `obs` and `obsm`) or variables (`var` and `varm`). `cellxgene launch` makes the following assumptions about your data (we recommend loading and inspecting your data using `scanpy` to validate these assumptions)
+
+- an `obs` field has a unique identifier for every cell (you can specify which field to use with the `--obs-names` option, by default it will use the value of `data.obs_names`)
+- a `var` field has a unique identifier for every gene (you can specify which field to use with the `--var-names` option, by default it will use the value of `data.var_names`)
+- an `obsm` field contains the two-dimensional coordinates for the layout that you want to render (e.g. `X_tsne` for the `tsne` layout or `X_umap` for the `umap` layout)
+- any additional `obs` fields will be rendered as per-cell continuous or categorical metadata by the app (e.g. `louvain` cluster assignments)
+
+### prepare
+
+The `prepare` command is included to help you format your data. It uses `scanpy` under the hood. This is especially useful if you are starting with raw unanalyzed data and are unfamiliar with `scanpy`.
+
+To prepare from an existing `.h5ad` file use
+
+```
+cellxgene prepare dataset.h5ad --output=dataset-processed.h5ad
+```
+
+This will load the input data, perform PCA and nearest neighbor calculations, compute `umap` and `tsne` layouts and `louvain` cluster assignments, and save the results in a new file called `dataset-processed.h5ad` that can be loaded using `cellxgene launch`. Data can be loaded from several formats, including `.h5ad` `.loom` and a `10-Genomics-formatted` `mtx` directory. Several options are available, including running one of the preprocessing `recipes` included with `scanpy`, which include steps like cell filtering and gene selection. To learn more about the `recipes` please see the `scanpy` [documentation](https://scanpy.readthedocs.io/en/latest/api/index.html#recipes).
+
+Depending on the options chosen, `prepare` can take a long time to run (a few minutes for datasets with 10-100k cells, up to an hour or more for datasets with >100k cells). If you want `prepare` to run faster we recommend using the `sparse` option and only computing the layout for `umap`, using a call like this
+
+```
+cellxgene prepare dataset.h5ad --output=dataset-processed.h5ad --layout=umap --sparse
+```
+
+To see all options call
+
+```
+cellxgene prepare --help
+```
+
+**Note**: `cellxgene prepare` will only perform `louvain` clustering if you have the `python-igraph` and `louvain` packages installed. To make sure they are installed alongside `cellxgene` use
+
+```
+pip install cellxgene[louvain]
+```
+
+If the aforementioned optional package installation fails, you can also install these packages directly:
+
+```
+pip install python-igraph louvain>=0.6
+```
+
+## conda and virtual environments
+
+If you use conda and want to create a conda environment for `cellxgene` you can use the following commands
+
+```
+conda create --yes -n cellxgene python=3.6
+conda activate cellxgene
+pip install cellxgene
+```
+
+Or you can create a virtual environment by using
+
+```
+ENV_NAME=cellxgene
+python3.6 -m venv ${ENV_NAME}
+source ${ENV_NAME}/bin/activate
+pip install cellxgene
+```
+
+## docker
+
+We have included a dockerfile to conveniently run cellxgene from docker.
+
+1. Build the image `docker build . -t cellxgene`
+2. Run the container and mount data `docker run -v "$PWD/example-dataset/:/data/" -p 5005:5005 cellxgene launch --host 0.0.0.0 data/pbmc3k.h5ad`
+   - You will need to use --host 0.0.0.0 to have the container listen to incoming requests from the browser
+
+## FAQ
+
+<details>
+
+<summary> questions about data formatting </summary>
+  
+<hr>
+  
+> Someone sent me a directory of `10X-Genomics` data with a `mtx` file and I've never used `scanpy`, can I use `cellxgene`?
+
+Yep! This should only take a couple steps. We'll assume your data is in a folder called `data/` and you've successfully installed `cellxgene` with the `louvain` packages as described above. Just run
+
+```
+cellxgene prepare data/ --output=data-processed.h5ad --layout=umap
+```
+
+Depending on the size of the dataset, this may take some time. Once it's done, call
+
+```
+cellxgene launch data-processed.h5ad --layout=umap --open
+```
+
+And your web browser should open with an interactive view of your data.
+
+<hr>
+
+> In my `prepare` command I received the following error `Warning: louvain module is not installed, no clusters will be calculated. To fix this please install cellxgene with the optional feature louvain enabled`
+
+Louvain clustering requires additional dependencies that are somewhat complex, so we don't include them by default. For now, you need to specify that you want these packages by using
+
+```
+pip install cellxgene[louvain]
+```
+
+<hr>
+
+> I ran `prepare` and I'm getting results that look unexpected
+
+You might want to try running one of the preprocessing recipes included with `scanpy` (read more about them [here](https://scanpy.readthedocs.io/en/latest/api/index.html#recipes)). You can specify this with the `--recipe` option, such as
+
+```
+cellxgene prepare data/ --output=data-processed.h5ad --recipe=zheng17
+```
+
+It should be easy to run `prepare` then call `cellxgene launch` a few times with different settings to explore different behaviors. We may explore adding other preprocessing options in the future.
+
+<hr>
+
+> I have extra metadata that I want to add to my dataset
+
+Currently this is not supported directly, but you should be able to do this manually using `scanpy`. For example, this [notebook](https://github.com/falexwolf/fun-analyses/blob/master/tabula_muris/tabula_muris.ipynb) shows adding the contents of a `csv` file with metadata to an `anndata` object. For now, you could do this manually on your data in the same way and then save out the result before loading into `cellxgene`.
+
+<hr>
+
+> What part of the anndata objects does cellxgene pull in for visualization?
+
+- `.obs` and `.var` annotations are use to extract metadata for filtering
+- `.X` is used to display expression (histograms, scatterplot & colorscale) and to compute differential expression
+- `.obsm` is used for layout
+
+</details>
+
+<details>
+  
+<summary> questions about installing and building </summary>
+
+<hr>
+
+> I tried to `pip install cellxgene` and got a weird error about missing paths to an HDF5 library?  
+
+You probably just need to install HDF5 first. If you're on a mac, you can simply `brew install hdf5` and then try `pip install cellxgene` again.  
+
+> I tried to `pip install cellxgene` and got another weird error I don't understand  
+
+This may happen, especially as we work out bugs in our installation process! Please create a new [Github issue](https://github.com/chanzuckerberg/cellxgene/issues), explain what you did, and include all the error messages you saw. It'd also be super helpful if you call `pip freeze` and include the full output alongside your issue.  
+
+<hr>
+
+> I'm following the developer instructions and get an error about "missing files and directories” when trying to build the client
+
+This is likely because you do not have node and npm installed, we recommend using [nvm](https://github.com/creationix/nvm) if you're new to using these tools.
+
+</details>
+
+<details>
+
+<summary> questions about algorithms </summary>
+
+<hr>
+
+> How are you computing and sorting differential expression results?
+
+Currently we use a [Welch's _t_-test](https://en.wikipedia.org/wiki/Welch%27s_t-test) implementation including the same variance overestimation correction as used in `scanpy`. We sort the `tscore` to identify the top N genes, and then filter to remove any that fall below a cutoff log fold change value, which can help remove spurious test results. The default threshold is `0.01` and can be changed using the option `--diffexp-lfc-cutoff`. We can explore adding support for other test types in the future.
+
+</details>
+
+## developer guide
+
+This project has made a few key design choices
+
+- The front-end is built with [`regl`](https://github.com/regl-project/regl) (a webgl library), [`react`](https://reactjs.org/), [`redux`](https://redux.js.org/), [`d3`](https://github.com/d3/d3), and [`blueprint`](https://blueprintjs.com/docs/#core) to handle rendering large numbers of cells with lots of complex interactivity
+- The app is designed with a client-server model that can support a range of existing analysis packages for backend computational tasks (currently built for [scanpy](https://github.com/theislab/scanpy))
+- The client uses fast cross-filtering to handle selections and comparisons across subsets of data
+
+Depending on your background and interests, you might want to contribute to the frontend, or backend, or both!
+
+If you are interested in working on `cellxgene` development, we recommend cloning the project from Gitub. First you'll need the following installed on your machine
+
+- python 3.6
+- node and npm (we recommend using [nvm](https://github.com/creationix/nvm) if this is your first time with node)
+
+Then clone the project
+
+```
+git clone https://github.com/chanzuckerberg/cellxgene.git
+```
+
+Build the client web assets by calling this from inside the `cellxgene` folder
+
+```
+./bin/build-client
+```
+
+Install all requirements (we recommend doing this inside a virtual environment)
+
+```
+pip install -e .
+```
+
+You can start the app while developing either by calling `cellxgene` or by calling `python -m server`. We recommend using the `--debug` flag to see more output, which you can include when reporting bugs.
+
+If you have any questions about developing or contributing, come hang out with us by joining the [CZI Science Slack](https://join-cellxgene-users.herokuapp.com/) and posting in the `#cellxgene-dev` channel.
+
+## development roadmap
+
+`cellxgene` is still very much in development, and we've love to include the community as we plan new features to work on. We are thinking about working on the following features over the next 3-12 months. If you are interested in updates, want to give feedback, want to contribute, or have ideas about other features we should work on, please [contact us](#help-and-contact)
+
+- **Visualizaling spatial metadata** Image-based transcriptomics methods also generate large cell by gene matrices, alongside rich metadata about spatial location; we would like to render this information in `cellxgene`
+- **Visualizing trajectories** Trajectory analyses infer progression along some ordering or pseudotime; we would like `cellxgene` to render the results of these analyses when they have been performed
+- **Deploy to web** Many projects release public data browser websites alongside their publicatons; we would like to make it easy for anyone to deploy `cellxgene` to a custom URL with their own dataset that they own and operate
+- **HCA Integration** The [Human Cell Atlas](https://humancellatlas.org) is generating a large corpus of single-cell expression data and will make it available through the Data Coordination Platform; we would like `cellxgene` to be one of several different portals for browsing these data
+
+## contributing
+
+We warmly welcome contributions from the community! Please submit any bug reports and feature requests through [Github issues](https://github.com/chanzuckerberg/cellxgene/issues). Please submit any direct contributions by forking the repository, creating a branch, and submitting a Pull Request. It'd be great for PRs to include test cases and documentation updates where relevant, though we know the core test suite is itself still a work in progress. And all code contributions and dependencies must be compatible with the project's open-source license (MIT). If you have any questions about this stuff, just ask!
+
+## inspiration and collaboration
+
+We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browswer](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [Gene Pattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data.
+
+We were inspired by Mike Bostock and the [crossfilter](https://github.com/crossfilter) team for the design of our filtering implementation.
+
+We have been working closely with the [`scanpy`](https://github.com/theislab/scanpy) team to integrate with their awesome analysis tools. Special thanks to Alex Wolf, Fabian Theis, and the rest of the team for their help during development and for providing an example dataset.
+
+We are eager to explore integrations with other computational backends such as [`Seurat`](https://github.com/satijalab/seurat) or [`Bioconductor`](https://github.com/Bioconductor)
+
+## help and contact
+
+Have questions, suggestions, or comments? You can come hang out with us by joining the [CZI Science Slack](https://join-cellxgene-users.herokuapp.com/) and posting in the `#cellxgene-users` channel. As mentioned above, please submit any feature requests or bugs as [Github issues](https://github.com/chanzuckerberg/cellxgene/issues). We'd love to hear from you!
+
+## reuse
+
+This project was started with the sole goal of empowering the scientific community to explore and understand their data. As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from this project, and reach out to us with ideas or questions. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).

client/__tests__/e2e/e2e.test.js

@@ -0,0 +1,185 @@
+import puppeteer from "puppeteer";
+
+const jest_env = process.env.JEST_ENV || "dev";
+const appPort = process.env.JEST_CXG_PORT || 3000;
+const appUrlBase = `http://localhost:${appPort}`;
+const DEV = jest_env === "dev";
+
+let browser;
+let page;
+const browserViewport = { width: 1280, height: 960 };
+
+beforeAll(async () => {
+  const browser_params = DEV
+    ? { headless: false, slowMo: 100, devtools: true }
+    : {};
+  browser = await puppeteer.launch(browser_params);
+  page = await browser.newPage();
+  page.setViewport(browserViewport);
+  if (DEV) page.on("console", msg => console.log("PAGE LOG:", msg.text()));
+});
+
+afterAll(() => {
+  if (!DEV) {
+    browser.close();
+  }
+});
+
+const getOneElementInnerHTML = async function(selector) {
+  let text = await page.$eval(selector, el => el.innerHTML);
+  return text;
+};
+
+const drag = async function(el_box, start, end, lasso = false) {
+  const x1 = el_box.content[0].x + start.x;
+  const x2 = el_box.content[0].x + end.x;
+  const y1 = el_box.content[0].y + start.y;
+  const y2 = el_box.content[0].y + end.y;
+  await page.mouse.move(x1, y1);
+  await page.mouse.down();
+  if (lasso) {
+    await page.mouse.move(x2, y1);
+    await page.mouse.move(x2, y2);
+    await page.mouse.move(x1, y2);
+    await page.mouse.move(x1, y1);
+  } else {
+    await page.mouse.move(x2, y2);
+  }
+  await page.mouse.up();
+};
+
+describe("did launch", () => {
+  test("page launched", async () => {
+    await page.goto(appUrlBase);
+    let el = await getOneElementInnerHTML("[data-testid='header']");
+    expect(el).toBe("cellxgene: pbmc3k");
+  });
+});
+
+describe("search for genes", () => {
+  test("search for known gene and add to metadata", async () => {
+    await page.goto(appUrlBase);
+    await page.waitForSelector("[ data-testid='gene-search']");
+    // blueprint's  typeahead is treating typing weird, clicking & waiting first solves this
+    await page.click("[data-testid='gene-search']");
+    await page.waitFor(200);
+    await page.type("[data-testid='gene-search']", "ACD");
+    await page.keyboard.press("Enter");
+    await page.waitForSelector("[data-testid='histogram-ACD']");
+  });
+});
+
+describe("select cells and diffexp", () => {
+  test("selects cells from layout and adds to cell set 1", async () => {
+    await page.goto(appUrlBase);
+    const layout = await page.waitForSelector("[data-testid='layout']");
+    const size = await layout.boxModel();
+    const cellset1 = {
+      start: {
+        x: Math.floor(size.width * 0.25),
+        y: Math.floor(size.height * 0.25)
+      },
+      end: {
+        x: Math.floor(size.width * 0.35),
+        y: Math.floor(size.height * 0.35)
+      }
+    };
+    await drag(size, cellset1.start, cellset1.end, true);
+    await page.click("[data-testid='cellset-button-1");
+    let button = await getOneElementInnerHTML("[data-testid='cellset-button-1");
+    expect(button).toMatch(/26 cells/);
+  });
+
+  test("selects cells from layout and adds to cell set 2", async () => {
+    await page.goto(appUrlBase);
+    const layout = await page.waitForSelector("[data-testid='layout']");
+    const size = await layout.boxModel();
+    const cellset2 = {
+      start: {
+        x: Math.floor(size.width * 0.45),
+        y: Math.floor(size.height * 0.45)
+      },
+      end: {
+        x: Math.floor(size.width * 0.55),
+        y: Math.floor(size.height * 0.55)
+      }
+    };
+    await drag(size, cellset2.start, cellset2.end, true);
+    await page.click("[data-testid='cellset-button-2");
+    let button = await getOneElementInnerHTML("[data-testid='cellset-button-2");
+    expect(button).toMatch(/49 cells/);
+  });
+
+  test("selects cells, saves them and performs diffexp", async () => {
+    await page.goto(appUrlBase);
+    const layout = await page.waitForSelector("[data-testid='layout']");
+    const size = await layout.boxModel();
+    const cellset1 = {
+      start: {
+        x: Math.floor(size.width * 0.25),
+        y: Math.floor(size.height * 0.25)
+      },
+      end: {
+        x: Math.floor(size.width * 0.35),
+        y: Math.floor(size.height * 0.35)
+      }
+    };
+    await drag(size, cellset1.start, cellset1.end, true);
+    await page.click("[data-testid='cellset-button-1");
+    const cellset2 = {
+      start: {
+        x: Math.floor(size.width * 0.45),
+        y: Math.floor(size.height * 0.45)
+      },
+      end: {
+        x: Math.floor(size.width * 0.55),
+        y: Math.floor(size.height * 0.55)
+      }
+    };
+    await drag(size, cellset2.start, cellset2.end, true);
+    await page.click("[data-testid='cellset-button-2");
+    await page.click("[data-testid='diffexp-button");
+    await page.waitForSelector("[data-testclass='histogram-diffexp']");
+    const diffexps = await page.$$eval(
+      "[data-testclass='histogram-diffexp']",
+      divs => {
+        return divs.map(div =>
+          div.id.substring("histogram-".length, div.id.length)
+        );
+      }
+    );
+    expect(diffexps).toMatchObject([
+      "HLA-DPA1",
+      "HLA-DQA1",
+      "HLA-DRB1",
+      "HLA-DMA",
+      "CST3",
+      "HLA-DPB1",
+      "HLA-DQB1",
+      "LGALS2",
+      "FCER1A",
+      "LTB"
+    ]);
+  });
+});
+
+describe("brushable histogram", () => {
+  test("can brush historgram", async () => {
+    await page.goto(appUrlBase);
+    const hist = await page.waitForSelector(
+      "[data-testid='histogram_n_genes_svg-brush'] > .overlay"
+    );
+    const hist_size = await hist.boxModel();
+    const draghist = {
+      start: {
+        x: Math.floor(hist_size.width * 0.25),
+        y: Math.floor(hist_size.height * 0.5)
+      },
+      end: {
+        x: Math.floor(hist_size.width * 0.55),
+        y: Math.floor(hist_size.height * 0.5)
+      }
+    };
+    await drag(hist_size, draghist.start, draghist.end);
+  });
+});

client/__tests__/setupMissingGlobals.js

@@ -0,0 +1,9 @@
+/*
+Define globals which are present in the client, but not in node (and therefore not in
+the jest test environment).
+*/
+
+import { TextDecoder, TextEncoder } from "util";
+
+global.TextDecoder = TextDecoder;
+global.TextEncoder = TextEncoder;

client/__tests__/util/actionHelpers.test.js

@@ -0,0 +1,60 @@
+import { rangeEncodeIndices } from "../../src/util/actionHelpers";
+
+describe("rangeEncodeIndices", () => {
+  test("small array edge cases", () => {
+    expect(rangeEncodeIndices([])).toMatchObject([]);
+    expect(rangeEncodeIndices([1])).toMatchObject([1]);
+    expect(rangeEncodeIndices([1, 99])).toMatchObject([1, 99]);
+    expect(rangeEncodeIndices([99, 1])).toMatchObject([1, 99]);
+    expect(rangeEncodeIndices([1, 100, 4])).toMatchObject([1, 4, 100]);
+  });
+
+  test("sorted flag", () => {
+    expect(rangeEncodeIndices([1, 9, 432], 10, true)).toMatchObject([
+      1,
+      9,
+      432
+    ]);
+    expect(rangeEncodeIndices([1, 9, 432], 10, false)).toMatchObject([
+      1,
+      9,
+      432
+    ]);
+    expect(rangeEncodeIndices([0, 1, 2, 3, 9, 10, 432], 2, true)).toMatchObject(
+      [[0, 3], [9, 10], 432]
+    );
+    expect(
+      rangeEncodeIndices([0, 1, 2, 3, 9, 10, 432], 2, false)
+    ).toMatchObject([[0, 3], [9, 10], 432]);
+  });
+
+  test("begin or end edge cases", () => {
+    expect(
+      rangeEncodeIndices([3, 4, 5, 6, 7, 10, 11, 12, 99], 3, false)
+    ).toMatchObject([[3, 7], [10, 12], 99]);
+    expect(
+      rangeEncodeIndices([3, 4, 5, 6, 7, 10, 11, 12], 3, false)
+    ).toMatchObject([[3, 7], [10, 12]]);
+    expect(
+      rangeEncodeIndices([0, 3, 4, 5, 6, 7, 10, 11, 12], 3, false)
+    ).toMatchObject([0, [3, 7], [10, 12]]);
+    expect(
+      rangeEncodeIndices([0, 3, 4, 5, 6, 7, 10, 11, 12, 99], 3, false)
+    ).toMatchObject([0, [3, 7], [10, 12], 99]);
+  });
+
+  test("minRangeLength", () => {
+    expect(
+      rangeEncodeIndices([3, 4, 5, 6, 7, 10, 11, 12, 99], 4, false)
+    ).toMatchObject([[3, 7], 10, 11, 12, 99]);
+    expect(
+      rangeEncodeIndices([3, 4, 5, 6, 7, 10, 11, 12], 4, false)
+    ).toMatchObject([[3, 7], 10, 11, 12]);
+    expect(
+      rangeEncodeIndices([0, 3, 4, 5, 6, 7, 10, 11, 12], 4, false)
+    ).toMatchObject([0, [3, 7], 10, 11, 12]);
+    expect(
+      rangeEncodeIndices([0, 3, 4, 5, 6, 7, 10, 11, 12, 99], 4, false)
+    ).toMatchObject([0, [3, 7], 10, 11, 12, 99]);
+  });
+});

client/__tests__/util/dataframe/dataframe.test.js

@@ -0,0 +1,616 @@
+import * as Dataframe from "../../../src/util/dataframe";
+
+describe("dataframe constructor", () => {
+  test("empty dataframe", () => {
+    const df = new Dataframe.Dataframe([0, 0], []);
+    expect(df).toBeDefined();
+    expect(df.dims).toEqual([0, 0]);
+    expect(df).toHaveLength(0);
+    expect(df.icol(0)).not.toBeDefined();
+  });
+
+  test("create with default indices", () => {
+    const df = new Dataframe.Dataframe(
+      [3, 2],
+      [new Int32Array(3).fill(0), new Int32Array(3).fill(1)]
+    );
+
+    expect(df).toBeDefined();
+    expect(df.dims).toEqual([3, 2]);
+    expect(df.rowIndex).toBeInstanceOf(Dataframe.IdentityInt32Index);
+    expect(df.colIndex).toBeInstanceOf(Dataframe.IdentityInt32Index);
+    expect(df.at(0, 0)).toEqual(0);
+    expect(df.at(2, 1)).toEqual(1);
+    expect(df.iat(0, 0)).toEqual(0);
+    expect(df.iat(2, 1)).toEqual(1);
+  });
+
+  test("create with labelled indices", () => {
+    const df = new Dataframe.Dataframe(
+      [3, 2],
+      [new Int32Array([0, 1, 2]), new Int32Array([3, 4, 5])],
+      new Dataframe.DenseInt32Index([2, 1, 0]),
+      new Dataframe.KeyIndex(["A", "B"])
+    );
+
+    expect(df).toBeDefined();
+    expect(df.dims).toEqual([3, 2]);
+
+    expect(df.rowIndex).toBeInstanceOf(Dataframe.DenseInt32Index);
+    expect(df.colIndex).toBeInstanceOf(Dataframe.KeyIndex);
+    expect(df.rowIndex.keys()).toEqual(new Int32Array([2, 1, 0]));
+    expect(df.colIndex.keys()).toEqual(["A", "B"]);
+
+    expect(df.at(0, "A")).toEqual(2);
+    expect(df.at(2, "B")).toEqual(3);
+    expect(df.iat(0, 0)).toEqual(0);
+    expect(df.iat(2, 1)).toEqual(5);
+  });
+});
+
+describe("simple data access", () => {
+  const df = new Dataframe.Dataframe(
+    [4, 2],
+    [
+      new Float64Array([0.0, Number.NaN, Number.POSITIVE_INFINITY, 3.14159]),
+      ["red", "blue", "green", "nan"]
+    ],
+    new Dataframe.DenseInt32Index([3, 2, 1, 0]),
+    new Dataframe.KeyIndex(["numbers", "colors"])
+  );
+
+  test("iat", () => {
+    expect(df).toBeDefined();
+
+    // present
+    expect(df.iat(0, 0)).toEqual(0.0);
+    expect(df.iat(0, 1)).toEqual("red");
+    expect(df.iat(1, 0)).toEqual(Number.NaN);
+    expect(df.iat(1, 1)).toEqual("blue");
+    expect(df.iat(2, 0)).toEqual(Number.POSITIVE_INFINITY);
+    expect(df.iat(2, 1)).toEqual("green");
+    expect(df.iat(3, 0)).toEqual(3.14159);
+    expect(df.iat(3, 1)).toEqual("nan");
+
+    // labels out of range have no defined behavior
+  });
+
+  test("at", () => {
+    expect(df).toBeDefined();
+
+    // present
+    expect(df.at(3, "numbers")).toEqual(0.0);
+    expect(df.at(3, "colors")).toEqual("red");
+    expect(df.at(2, "numbers")).toEqual(Number.NaN);
+    expect(df.at(2, "colors")).toEqual("blue");
+    expect(df.at(1, "numbers")).toEqual(Number.POSITIVE_INFINITY);
+    expect(df.at(1, "colors")).toEqual("green");
+    expect(df.at(0, "numbers")).toEqual(3.14159);
+    expect(df.at(0, "colors")).toEqual("nan");
+
+    // labels out of range have no defined behavior
+  });
+
+  test("ihas", () => {
+    expect(df).toBeDefined();
+
+    // present
+    expect(df.ihas(0, 0)).toBeTruthy();
+    expect(df.ihas(1, 1)).toBeTruthy();
+    expect(df.ihas(3, 1)).toBeTruthy();
+
+    // not present
+    expect(df.ihas(-1, -1)).toBeFalsy();
+    expect(df.ihas(0, 99)).toBeFalsy();
+    expect(df.ihas(99, 0)).toBeFalsy();
+    expect(df.ihas(99, 99)).toBeFalsy();
+    expect(df.ihas(-1, 0)).toBeFalsy();
+    expect(df.ihas(0, -1)).toBeFalsy();
+  });
+
+  test("has", () => {
+    expect(df).toBeDefined();
+
+    // present
+    expect(df.has(3, "numbers")).toBeTruthy();
+    expect(df.has(0, "numbers")).toBeTruthy();
+    expect(df.has(3, "colors")).toBeTruthy();
+    expect(df.has(0, "colors")).toBeTruthy();
+
+    // not present
+    expect(df.has(3, "foo")).toBeFalsy();
+    expect(df.has(-1, "numbers")).toBeFalsy();
+    expect(df.has(-1, -1)).toBeFalsy();
+    expect(df.has(null, null)).toBeFalsy();
+    expect(df.has(0, "foo")).toBeFalsy();
+    expect(df.has(99, "numbers")).toBeFalsy();
+    expect(df.has(99, "foo")).toBeFalsy();
+  });
+});
+
+describe("dataframe subsetting", () => {
+  describe("subset", () => {
+    const sourceDf = new Dataframe.Dataframe(
+      [3, 4],
+      [
+        new Int32Array([0, 1, 2]),
+        ["A", "B", "C"],
+        new Float32Array([4.4, 5.5, 6.6]),
+        ["red", "green", "blue"]
+      ],
+      null,
+      new Dataframe.KeyIndex(["int32", "string", "float32", "colors"])
+    );
+
+    test("all rows, one column", () => {
+      const dfA = sourceDf.subset(null, ["colors"]);
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([3, 1]);
+      expect(dfA.iat(0, 0)).toEqual("red");
+      expect(dfA.at(2, "colors")).toEqual("blue");
+      expect(dfA.col("colors").asArray()).toEqual(["red", "green", "blue"]);
+      expect(dfA.icol(0).asArray()).toEqual(["red", "green", "blue"]);
+      expect(dfA.col("colors").asArray()).toEqual(
+        sourceDf.col("colors").asArray()
+      );
+      expect(dfA.rowIndex.keys()).toEqual(sourceDf.rowIndex.keys());
+      expect(dfA.colIndex.keys()).toEqual(["colors"]);
+    });
+
+    test("all rows, two columns", () => {
+      const dfB = sourceDf.subset(null, ["colors", "float32"]);
+      expect(dfB).toBeDefined();
+      expect(dfB.dims).toEqual([3, 2]);
+      expect(dfB.iat(0, 0)).toBeCloseTo(4.4);
+      expect(dfB.iat(0, 1)).toEqual("red");
+      expect(dfB.at(2, "colors")).toEqual("blue");
+      expect(dfB.at(2, "float32")).toBeCloseTo(6.6);
+      expect(dfB.col("colors").asArray()).toEqual(["red", "green", "blue"]);
+      expect(dfB.col("float32").asArray()).toEqual(
+        new Float32Array([4.4, 5.5, 6.6])
+      );
+      expect(dfB.icol(0).asArray()).toEqual(dfB.col("float32").asArray());
+      expect(dfB.icol(1).asArray()).toEqual(dfB.col("colors").asArray());
+      expect(dfB.col("colors").asArray()).toEqual(
+        sourceDf.col("colors").asArray()
+      );
+      expect(dfB.col("float32").asArray()).toEqual(
+        sourceDf.col("float32").asArray()
+      );
+      expect(dfB.rowIndex.keys()).toEqual(sourceDf.rowIndex.keys());
+      expect(dfB.colIndex.keys()).toEqual(["float32", "colors"]);
+    });
+
+    test("one row, all columns", () => {
+      const dfC = sourceDf.subset([1], null);
+      expect(dfC).toBeDefined();
+      expect(dfC.dims).toEqual([1, 4]);
+      expect(dfC.iat(0, 0)).toEqual(1);
+      expect(dfC.iat(0, 1)).toEqual("B");
+      expect(dfC.iat(0, 2)).toBeCloseTo(5.5);
+      expect(dfC.iat(0, 3)).toEqual("green");
+      expect(dfC.rowIndex.keys()).toEqual(new Int32Array([1]));
+      expect(dfC.colIndex.keys()).toEqual(sourceDf.colIndex.keys());
+    });
+
+    test("two rows, all columns", () => {
+      const dfD = sourceDf.subset([0, 2], null);
+      expect(dfD).toBeDefined();
+      expect(dfD.dims).toEqual([2, 4]);
+      expect(dfD.icol(0).asArray()).toEqual(new Int32Array([0, 2]));
+      expect(dfD.icol(1).asArray()).toEqual(["A", "C"]);
+      expect(dfD.icol(2).asArray()).toEqual(new Float32Array([4.4, 6.6]));
+      expect(dfD.icol(3).asArray()).toEqual(["red", "blue"]);
+      expect(dfD.rowIndex.keys()).toEqual(new Int32Array([0, 2]));
+      expect(dfD.colIndex.keys()).toEqual(sourceDf.colIndex.keys());
+    });
+
+    test("all rows, all columns", () => {
+      const dfE = sourceDf.subset(null, null);
+      expect(dfE).toBeDefined();
+      expect(dfE.dims).toEqual([3, 4]);
+      expect(dfE.icol(0).asArray()).toEqual(sourceDf.icol(0).asArray());
+      expect(dfE.icol(1).asArray()).toEqual(sourceDf.icol(1).asArray());
+      expect(dfE.icol(2).asArray()).toEqual(sourceDf.icol(2).asArray());
+      expect(dfE.icol(3).asArray()).toEqual(sourceDf.icol(3).asArray());
+      expect(dfE.rowIndex.keys()).toEqual(sourceDf.rowIndex.keys());
+      expect(dfE.colIndex.keys()).toEqual(sourceDf.colIndex.keys());
+    });
+
+    test("two rows, two colums", () => {
+      const dfF = sourceDf.subset([0, 2], ["int32", "float32"]);
+      expect(dfF).toBeDefined();
+      expect(dfF.dims).toEqual([2, 2]);
+      expect(dfF.icol(0).asArray()).toEqual(new Int32Array([0, 2]));
+      expect(dfF.icol(1).asArray()).toEqual(new Float32Array([4.4, 6.6]));
+      expect(dfF.rowIndex.keys()).toEqual(new Int32Array([0, 2]));
+      expect(dfF.colIndex.keys()).toEqual(["int32", "float32"]);
+    });
+
+    test("withRowIndex", () => {
+      const df = sourceDf.subset(
+        null,
+        ["int32", "float32"],
+        new Dataframe.DenseInt32Index([3, 2, 1])
+      );
+      expect(df.colIndex).toBeInstanceOf(Dataframe.KeyIndex);
+      expect(df.rowIndex).toBeInstanceOf(Dataframe.DenseInt32Index);
+      expect(df.at(3, "int32")).toEqual(df.iat(0, 0));
+    });
+
+    test("withRowIndex error checks", () => {
+      expect(() =>
+        sourceDf.subset(null, ["red"], new Dataframe.IdentityInt32Index(1))
+      ).toThrow(RangeError);
+      expect(() =>
+        sourceDf.subset(null, ["red"], new Dataframe.DenseInt32Index([0, 1]))
+      ).toThrow(RangeError);
+      expect(() =>
+        sourceDf.subset(null, ["red"], new Dataframe.KeyIndex([0, 1, 2, 3]))
+      ).toThrow(RangeError);
+    });
+  });
+
+  test("isubsetMask", () => {
+    const sourceDf = new Dataframe.Dataframe(
+      [3, 4],
+      [
+        new Int32Array([0, 1, 2]),
+        ["A", "B", "C"],
+        new Float32Array([4.4, 5.5, 6.6]),
+        ["red", "green", "blue"]
+      ],
+      new Dataframe.DenseInt32Index([2, 4, 6]),
+      new Dataframe.KeyIndex(["int32", "string", "float32", "colors"])
+    );
+
+    const dfA = sourceDf.isubsetMask(
+      new Uint8Array([0, 1, 1]),
+      new Uint8Array([1, 0, 0, 1])
+    );
+    expect(dfA.dims).toEqual([2, 2]);
+    expect(dfA.icol(0).asArray()).toEqual(new Int32Array([1, 2]));
+    expect(dfA.icol(1).asArray()).toEqual(["green", "blue"]);
+    expect(dfA.rowIndex.keys()).toEqual(new Int32Array([4, 6]));
+    expect(dfA.colIndex.keys()).toEqual(["int32", "colors"]);
+  });
+});
+
+describe("dataframe factories", () => {
+  test("create", () => {
+    const df = Dataframe.Dataframe.create(
+      [3, 3],
+      [
+        new Array(3).fill(0),
+        new Int16Array(3).fill(99),
+        new Float64Array(3).fill(1.1)
+      ]
+    );
+
+    expect(df).toBeDefined();
+    expect(df.dims).toEqual([3, 3]);
+    expect(df).toHaveLength(3);
+    expect(df.iat(0, 0)).toEqual(0);
+    expect(df.iat(1, 1)).toEqual(99);
+    expect(df.iat(2, 2)).toBeCloseTo(1.1);
+    expect(df.iat(0, 0)).toEqual(df.at(0, 0));
+    expect(df.iat(1, 1)).toEqual(df.at(1, 1));
+    expect(df.iat(2, 2)).toEqual(df.at(2, 2));
+  });
+
+  test("clone", () => {
+    const dfA = new Dataframe.Dataframe(
+      [3, 2],
+      [new Int32Array([0, 1, 2]), new Int32Array([3, 4, 5])],
+      new Dataframe.DenseInt32Index([2, 1, 0]),
+      new Dataframe.KeyIndex(["A", "B"])
+    );
+
+    const dfB = dfA.clone();
+    expect(dfB).not.toBe(dfA);
+    expect(dfB.dims).toEqual(dfA.dims);
+    expect(dfB).toHaveLength(dfA.length);
+    expect(dfB.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    expect(dfB.colIndex.keys()).toEqual(dfA.colIndex.keys());
+    for (let i = 0, l = dfB.dims[1]; i < l; i += 1) {
+      expect(dfB.icol(i).asArray()).toEqual(dfA.icol(i).asArray());
+    }
+  });
+
+  describe("withCol", () => {
+    test("KeyIndex", () => {
+      const df = new Dataframe.Dataframe(
+        [2, 2],
+        [["red", "blue"], [true, false]],
+        null,
+        new Dataframe.KeyIndex(["colors", "bools"])
+      );
+      const dfA = df.withCol("numbers", [1, 0]);
+
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([2, 3]);
+      expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.icol(1).asArray()).toEqual([true, false]);
+      expect(dfA.icol(2).asArray()).toEqual([1, 0]);
+      expect(dfA.col("numbers").asArray()).toEqual([1, 0]);
+      expect(dfA.colIndex.keys()).toEqual(["colors", "bools", "numbers"]);
+      expect(df.colIndex.keys()).toEqual(["colors", "bools"]);
+      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    });
+
+    test("DenseInt32Index", () => {
+      const df = new Dataframe.Dataframe(
+        [2, 2],
+        [["red", "blue"], [true, false]],
+        null,
+        new Dataframe.DenseInt32Index([74, 75])
+      );
+      const dfA = df.withCol(72, [1, 0]);
+
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([2, 3]);
+      expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.icol(1).asArray()).toEqual([true, false]);
+      expect(dfA.icol(2).asArray()).toEqual([1, 0]);
+      expect(dfA.col(74).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.col(75).asArray()).toEqual([true, false]);
+      expect(dfA.col(72).asArray()).toEqual([1, 0]);
+      expect(dfA.colIndex.keys()).toEqual(new Int32Array([74, 75, 72]));
+      expect(df.colIndex.keys()).toEqual(new Int32Array([74, 75]));
+      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    });
+
+    test("DenseInt32Index promote", () => {
+      const df = new Dataframe.Dataframe(
+        [2, 2],
+        [["red", "blue"], [true, false]],
+        null,
+        new Dataframe.DenseInt32Index([74, 75])
+      );
+      const dfA = df.withCol(999, [1, 0]);
+
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([2, 3]);
+      expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.icol(1).asArray()).toEqual([true, false]);
+      expect(dfA.icol(2).asArray()).toEqual([1, 0]);
+      expect(dfA.col(74).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.col(75).asArray()).toEqual([true, false]);
+      expect(dfA.col(999).asArray()).toEqual([1, 0]);
+      expect(dfA.colIndex.keys()).toEqual(new Int32Array([74, 75, 999]));
+      expect(df.colIndex.keys()).toEqual(new Int32Array([74, 75]));
+      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    });
+
+    test("IdentityInt32Index with last", () => {
+      const df = new Dataframe.Dataframe(
+        [2, 2],
+        [["red", "blue"], [true, false]],
+        null,
+        null
+      );
+      const dfA = df.withCol(2, [1, 0]);
+
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([2, 3]);
+      expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.icol(1).asArray()).toEqual([true, false]);
+      expect(dfA.icol(2).asArray()).toEqual([1, 0]);
+      expect(dfA.col(0).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.col(1).asArray()).toEqual([true, false]);
+      expect(dfA.col(2).asArray()).toEqual([1, 0]);
+      expect(dfA.colIndex.keys()).toEqual(new Int32Array([0, 1, 2]));
+      expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1]));
+      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    });
+
+    test("IdentityInt32Index promote", () => {
+      const df = new Dataframe.Dataframe(
+        [2, 2],
+        [["red", "blue"], [true, false]],
+        null,
+        null
+      );
+      const dfA = df.withCol(99, [1, 0]);
+
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([2, 3]);
+      expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.icol(1).asArray()).toEqual([true, false]);
+      expect(dfA.icol(2).asArray()).toEqual([1, 0]);
+      expect(dfA.col(0).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.col(1).asArray()).toEqual([true, false]);
+      expect(dfA.col(99).asArray()).toEqual([1, 0]);
+      expect(dfA.colIndex.keys()).toEqual(new Int32Array([0, 1, 99]));
+      expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1]));
+      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    });
+
+    describe("handle column dimensions correctly", () => {
+      /*
+      there are two conditions:
+      - empty dataframe - will accept an add of any dimensionality
+      - non-empty dataframe - added column must match row-count dimension
+      */
+      test("empty.withCol", () => {
+        const edf = Dataframe.Dataframe.empty();
+        const df = edf.withCol("foo", [1, 2, 3]);
+
+        expect(edf).toBeDefined();
+        expect(df).toBeDefined();
+        expect(edf).not.toEqual(df);
+        expect(df.dims).toEqual([3, 1]);
+        expect(df.icol(0).asArray()).toEqual([1, 2, 3]);
+      });
+
+      test("withCol dimension check", () => {
+        const dfA = new Dataframe.Dataframe([1, 1], [["a"]]);
+        expect(() => {
+          dfA.withCol(1, []);
+        }).toThrow(RangeError);
+      });
+    });
+  });
+
+  describe("dropCol", () => {
+    test("KeyIndex", () => {
+      const df = new Dataframe.Dataframe(
+        [2, 3],
+        [["red", "blue"], [true, false], [1, 0]],
+        null,
+        new Dataframe.KeyIndex(["colors", "bools", "numbers"])
+      );
+      const dfA = df.dropCol("colors");
+
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([2, 2]);
+      expect(dfA.icol(0).asArray()).toEqual([true, false]);
+      expect(dfA.icol(1).asArray()).toEqual([1, 0]);
+      expect(dfA.col("numbers").asArray()).toEqual([1, 0]);
+      expect(dfA.colIndex.keys()).toEqual(["bools", "numbers"]);
+      expect(df.colIndex.keys()).toEqual(["colors", "bools", "numbers"]);
+      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    });
+
+    test("IdentityInt32Index drop first", () => {
+      const df = new Dataframe.Dataframe(
+        [2, 3],
+        [["red", "blue"], [true, false], [1, 0]],
+        null,
+        null
+      );
+      const dfA = df.dropCol(0);
+
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([2, 2]);
+      expect(dfA.icol(0).asArray()).toEqual([true, false]);
+      expect(dfA.icol(1).asArray()).toEqual([1, 0]);
+      expect(df.col(1).asArray()).toEqual(dfA.col(1).asArray());
+      expect(df.col(2).asArray()).toEqual(dfA.col(2).asArray());
+      expect(dfA.colIndex.keys()).toEqual(new Int32Array([1, 2]));
+      expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1, 2]));
+      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    });
+
+    test("IdentityInt32Index drop last", () => {
+      const df = new Dataframe.Dataframe(
+        [2, 3],
+        [["red", "blue"], [true, false], [1, 0]],
+        null,
+        null
+      );
+      const dfA = df.dropCol(2);
+
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([2, 2]);
+      expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.icol(1).asArray()).toEqual([true, false]);
+      expect(df.col(0).asArray()).toEqual(dfA.col(0).asArray());
+      expect(df.col(1).asArray()).toEqual(dfA.col(1).asArray());
+      expect(dfA.colIndex.keys()).toEqual(new Int32Array([0, 1]));
+      expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1, 2]));
+      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    });
+
+    test("DenseInt32Index", () => {
+      const df = new Dataframe.Dataframe(
+        [2, 3],
+        [["red", "blue"], [true, false], [1, 0]],
+        null,
+        new Dataframe.DenseInt32Index([102, 101, 100])
+      );
+      const dfA = df.dropCol(101);
+
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([2, 2]);
+      expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.icol(1).asArray()).toEqual([1, 0]);
+      expect(dfA.col(100).asArray()).toEqual([1, 0]);
+      expect(dfA.col(102).asArray()).toEqual(["red", "blue"]);
+      expect(dfA.colIndex.keys()).toEqual(new Int32Array([102, 100]));
+      expect(df.colIndex.keys()).toEqual(new Int32Array([102, 101, 100]));
+      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    });
+  });
+});
+
+describe("dataframe col", () => {
+  let df = null;
+  beforeEach(() => {
+    df = new Dataframe.Dataframe(
+      [2, 2],
+      [[true, false], [1, 0]],
+      null,
+      new Dataframe.KeyIndex(["A", "B"])
+    );
+  });
+
+  test("col", () => {
+    expect(df).toBeDefined();
+    expect(df.col("A")).toBe(df.icol(0));
+    expect(df.col("B")).toBe(df.icol(1));
+    expect(df.col("undefined")).toBeUndefined();
+    expect(df.icol("undefined")).toBeUndefined();
+
+    const colA = df.col("A");
+    expect(colA).toBeInstanceOf(Function);
+    expect(colA.asArray).toBeInstanceOf(Function);
+    expect(colA.has).toBeInstanceOf(Function);
+    expect(colA.ihas).toBeInstanceOf(Function);
+    expect(colA.indexOf).toBeInstanceOf(Function);
+    expect(colA.iget).toBeInstanceOf(Function);
+  });
+
+  test("col.asArray", () => {
+    expect(df).toBeDefined();
+    expect(df.col("A").asArray()).toEqual([true, false]);
+    expect(df.icol(0).asArray()).toEqual([true, false]);
+    expect(df.col("B").asArray()).toEqual([1, 0]);
+    expect(df.icol(1).asArray()).toEqual([1, 0]);
+  });
+
+  test("col.has", () => {
+    expect(df).toBeDefined();
+    expect(df.col("A").has(-1)).toBe(false);
+    expect(df.col("A").has(0)).toBe(true);
+    expect(df.col("A").has(1)).toBe(true);
+    expect(df.col("A").has(2)).toBe(false);
+    expect(df.col("B").has(-1)).toBe(false);
+    expect(df.col("B").has(0)).toBe(true);
+    expect(df.col("B").has(1)).toBe(true);
+    expect(df.col("B").has(2)).toBe(false);
+  });
+
+  test("col.ihas", () => {
+    expect(df).toBeDefined();
+    expect(df.col("A").ihas(-1)).toBe(false);
+    expect(df.col("A").ihas(0)).toBe(true);
+    expect(df.col("A").ihas(1)).toBe(true);
+    expect(df.col("A").ihas(2)).toBe(false);
+    expect(df.col("B").ihas(-1)).toBe(false);
+    expect(df.col("B").ihas(0)).toBe(true);
+    expect(df.col("B").ihas(1)).toBe(true);
+    expect(df.col("B").ihas(2)).toBe(false);
+  });
+
+  test("col.iget", () => {
+    expect(df).toBeDefined();
+    expect(df.col("A").iget(0)).toEqual(df.iat(0, 0));
+    expect(df.col("B").iget(1)).toEqual(df.iat(1, 1));
+  });
+
+  test("col.indexOf", () => {
+    expect(df).toBeDefined();
+    expect(df.col("A").indexOf(true)).toEqual(0);
+    expect(df.col("A").indexOf(false)).toEqual(1);
+    expect(df.col("A").indexOf(99)).toBeUndefined();
+    expect(df.col("A").indexOf(undefined)).toBeUndefined();
+    expect(df.col("A").indexOf(1)).toBeUndefined();
+
+    expect(df.col("B").indexOf(1)).toEqual(0);
+    expect(df.col("B").indexOf(0)).toEqual(1);
+    expect(df.col("B").indexOf(99)).toBeUndefined();
+    expect(df.col("B").indexOf(undefined)).toBeUndefined();
+    expect(df.col("B").indexOf(true)).toBeUndefined();
+  });
+});

client/__tests__/util/dataframe/summarize.test.js

@@ -0,0 +1,253 @@
+import * as Dataframe from "../../../src/util/dataframe";
+
+function float32Conversion(f) {
+  return new Float32Array([f])[0];
+}
+
+describe("Dataframe column summary", () => {
+  test("empty column test", () => {
+    const df = Dataframe.Dataframe.create([0, 1], [[]]);
+    const summary = df.icol(0).summarize();
+    expect(summary).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: [],
+        categoryCounts: new Map(),
+        numCategories: 0
+      })
+    );
+  });
+
+  test("simple test", () => {
+    const df = new Dataframe.Dataframe(
+      [1, 6],
+      [
+        ["n1"],
+        ["hi"],
+        [true],
+        new Float32Array([39.3]),
+        new Int32Array([99]),
+        [1]
+      ],
+      null,
+      new Dataframe.KeyIndex([
+        "name",
+        "nameString",
+        "nameBoolean",
+        "nameFloat32",
+        "nameInt32",
+        "nameCategorical"
+      ])
+    );
+
+    expect(df.icol(0).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: ["n1"],
+        categoryCounts: new Map([["n1", 1]]),
+        numCategories: 1
+      })
+    );
+    expect(df.icol(1).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: ["hi"],
+        categoryCounts: new Map([["hi", 1]]),
+        numCategories: 1
+      })
+    );
+    expect(df.icol(2).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: [true],
+        categoryCounts: new Map([[true, 1]]),
+        numCategories: 1
+      })
+    );
+    expect(df.icol(3).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: false,
+        min: float32Conversion(39.3),
+        max: float32Conversion(39.3),
+        nan: 0,
+        ninf: 0,
+        pinf: 0
+      })
+    );
+    expect(df.icol(4).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: false,
+        min: 99,
+        max: 99,
+        nan: 0,
+        ninf: 0,
+        pinf: 0
+      })
+    );
+    expect(df.icol(5).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: [1],
+        categoryCounts: new Map([[1, 1]]),
+        numCategories: 1
+      })
+    );
+  });
+
+  test("multi test", () => {
+    const df = new Dataframe.Dataframe(
+      [3, 6],
+      [
+        ["n0", "n1", "n2"],
+        ["hi", "hi", "bye"],
+        [false, true, true],
+        new Float32Array([39.3, 39.3, 0]),
+        new Int32Array([99, 99, 99]),
+        [1, false, "0"]
+      ],
+      null,
+      new Dataframe.KeyIndex([
+        "name",
+        "nameString",
+        "nameBoolean",
+        "nameFloat32",
+        "nameInt32",
+        "nameCategorical"
+      ])
+    );
+
+    expect(df.icol(0).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining(["n0", "n1", "n2"]),
+        categoryCounts: new Map([["n0", 1], ["n1", 1], ["n2", 1]]),
+        numCategories: 3
+      })
+    );
+    expect(df.icol(1).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining(["hi", "bye"]),
+        categoryCounts: new Map([["hi", 2], ["bye", 1]]),
+        numCategories: 2
+      })
+    );
+    expect(df.icol(2).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining([true, false]),
+        categoryCounts: new Map([[true, 2], [false, 1]]),
+        numCategories: 2
+      })
+    );
+    expect(df.icol(3).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: false,
+        min: 0,
+        max: float32Conversion(39.3),
+        nan: 0,
+        ninf: 0,
+        pinf: 0
+      })
+    );
+    expect(df.icol(4).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: false,
+        min: 99,
+        max: 99,
+        nan: 0,
+        ninf: 0,
+        pinf: 0
+      })
+    );
+    expect(df.icol(5).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining([1, false, "0"]),
+        categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]),
+        numCategories: 3
+      })
+    );
+  });
+
+  test("non-finite numbers", () => {
+    const df = new Dataframe.Dataframe(
+      [4, 6],
+      [
+        ["n0", "n1", "n2", "n2"],
+        ["hi", "hi", "bye", "bye"],
+        [false, true, true, true],
+        new Float32Array([
+          39.3,
+          Number.NEGATIVE_INFINITY,
+          Number.NaN,
+          Number.POSITIVE_INFINITY
+        ]),
+        new Int32Array([99, 99, 99, 99]),
+        [1, false, "0", "0"]
+      ],
+      null,
+      new Dataframe.KeyIndex([
+        "name",
+        "nameString",
+        "nameBoolean",
+        "nameFloat32",
+        "nameInt32",
+        "nameCategorical"
+      ])
+    );
+
+    expect(df.icol(0).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining(["n0", "n1", "n2"]),
+        categoryCounts: new Map([["n0", 1], ["n1", 1], ["n2", 2]]),
+        numCategories: 3
+      })
+    );
+    expect(df.icol(1).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining(["hi", "bye"]),
+        categoryCounts: new Map([["hi", 2], ["bye", 1]]),
+        numCategories: 2
+      })
+    );
+    expect(df.icol(2).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining([true, false]),
+        categoryCounts: new Map([[true, 2], [false, 1]]),
+        numCategories: 2
+      })
+    );
+    expect(df.icol(3).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: false,
+        min: float32Conversion(39.3),
+        max: float32Conversion(39.3),
+        nan: 1,
+        ninf: 1,
+        pinf: 1
+      })
+    );
+    expect(df.icol(4).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: false,
+        min: 99,
+        max: 99,
+        nan: 0,
+        ninf: 0,
+        pinf: 0
+      })
+    );
+    expect(df.icol(5).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining([1, false, "0"]),
+        categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]),
+        numCategories: 3
+      })
+    );
+  });
+});

client/__tests__/util/nameCreators.test.js

@@ -0,0 +1,68 @@
+/*
+Test cases for nameCreators.js.
+*/
+import {
+  layoutDimensionName,
+  obsAnnoDimensionName,
+  diffexpDimensionName,
+  userDefinedDimensionName,
+  makeContinuousDimensionName
+} from "../../src/util/nameCreators";
+
+describe("nameCreators", () => {
+  const nameCreators = [
+    layoutDimensionName,
+    obsAnnoDimensionName,
+    diffexpDimensionName,
+    userDefinedDimensionName
+  ];
+
+  test("check for namespace isolation", () => {
+    const foo = "foo";
+    nameCreators.forEach(fn => expect(fn(foo)).not.toBe(foo));
+
+    const bar = "bar";
+    nameCreators.forEach(fn => expect(fn(foo)).not.toBe(fn(bar)));
+
+    nameCreators.forEach(fn => {
+      const allOtherFn = nameCreators.filter(elmnt => elmnt !== fn);
+      allOtherFn.forEach(otherFn => expect(fn(foo)).not.toBe(otherFn(foo)));
+    });
+  });
+
+  test("check for legal keys", () => {
+    /* need namespace creators to return strings only */
+    nameCreators.forEach(fn => expect(fn("X")).toMatch(/X/));
+  });
+});
+
+describe("makeContinuousDimensionName", () => {
+  test("namespaces", () => {
+    expect(makeContinuousDimensionName({ isObs: true }, "OK")).toMatch(/OK/);
+    expect(makeContinuousDimensionName({ isDiffExp: true }, "OK")).toMatch(
+      /OK/
+    );
+    expect(makeContinuousDimensionName({ isUserDefined: true }, "OK")).toMatch(
+      /OK/
+    );
+  });
+
+  test("error handling", () => {
+    expect(() => makeContinuousDimensionName({}, "oops")).toThrow();
+    expect(() =>
+      makeContinuousDimensionName(
+        { isObs: false, isDiffExp: false, isUserDefined: false },
+        "oops"
+      )
+    ).toThrow();
+    expect(() =>
+      makeContinuousDimensionName({ isObs: true }, "OK")
+    ).not.toThrow();
+    expect(() =>
+      makeContinuousDimensionName({ isDiffExp: true }, "OK")
+    ).not.toThrow();
+    expect(() =>
+      makeContinuousDimensionName({ isUserDefined: true }, "OK")
+    ).not.toThrow();
+  });
+});

client/__tests__/util/stateManager/sampleResponses.js

@@ -0,0 +1,205 @@
+/* eslint no-bitwise: "off" */
+import _ from "lodash";
+import { flatbuffers } from "flatbuffers";
+import { NetEncoding } from "../../../src/util/stateManager/matrix_generated";
+
+/*
+test data mocking REST 0.2 API responses.  Used in several tests.
+*/
+const nObs = 10;
+const nVar = 32;
+const field4Categories = [83, true, "foo", 2.222222];
+const fieldDCategories = [99, false, "mumble", 3.1415];
+
+const aConfigResponse = {
+  config: {
+    features: [
+      { method: "POST", path: "/cluster/", available: false },
+      { method: "POST", path: "/layout/", available: false },
+      { method: "POST", path: "/diffexp/", available: false },
+      { method: "POST", path: "/saveLocal/", available: false }
+    ],
+    displayNames: {
+      engine: "the little engine that could",
+      dataset: "all your zeros are mine"
+    }
+  }
+};
+
+const aSchemaResponse = {
+  schema: {
+    dataframe: {
+      nObs,
+      nVar,
+      type: "float32"
+    },
+    annotations: {
+      obs: [
+        { name: "name", type: "string" },
+        { name: "field1", type: "int32" },
+        { name: "field2", type: "float32" },
+        { name: "field3", type: "boolean" },
+        {
+          name: "field4",
+          type: "categorical",
+          categories: field4Categories
+        }
+      ],
+      var: [
+        { name: "name", type: "string" },
+        { name: "fieldA", type: "int32" },
+        { name: "fieldB", type: "float32" },
+        { name: "fieldC", type: "boolean" },
+        {
+          name: "fieldD",
+          type: "categorical",
+          categories: fieldDCategories
+        }
+      ]
+    }
+  }
+};
+
+const anAnnotationsObsJSONResponse = {
+  names: ["name", "field1", "field2", "field3", "field4"],
+  data: _()
+    .range(nObs)
+    .map(idx => [
+      idx,
+      `obs${idx}`,
+      2 * idx,
+      idx + 0.0133,
+      !!(idx & 1),
+      field4Categories[idx % field4Categories.length]
+    ])
+    .value()
+};
+
+const anAnnotationsVarJSONResponse = {
+  names: ["fieldA", "fieldB", "fieldC", "fieldD", "name"],
+  data: _()
+    .range(nVar)
+    .map(idx => [
+      idx,
+      10 * idx,
+      idx + 2.90143,
+      !!(idx & 1),
+      fieldDCategories[idx % fieldDCategories.length],
+      `var${idx}`
+    ])
+    .value()
+};
+
+function encodeTypedArray(builder, uType, uData) {
+  const uTypeName = NetEncoding.TypedArray[uType];
+  const ArrayType = NetEncoding[uTypeName];
+  const dv = ArrayType.createDataVector(builder, uData);
+  builder.startObject(1);
+  builder.addFieldOffset(0, dv, 0);
+  return builder.endObject();
+}
+
+function encodeMatrix(columns, colIndex = undefined) {
+  const utf8Encoder = new TextEncoder("utf-8");
+  const builder = new flatbuffers.Builder(1024);
+  const cols = _.map(columns, carr => {
+    let uType;
+    let tarr;
+    if (_.every(carr, _.isNumber)) {
+      uType = NetEncoding.TypedArray.Float32Array;
+      tarr = encodeTypedArray(builder, uType, new Float32Array(carr));
+    } else {
+      uType = NetEncoding.TypedArray.JSONEncodedArray;
+      const json = JSON.stringify(carr);
+      const jsonUTF8 = utf8Encoder.encode(json);
+      tarr = encodeTypedArray(builder, uType, jsonUTF8);
+    }
+    NetEncoding.Column.startColumn(builder);
+    NetEncoding.Column.addUType(builder, uType);
+    NetEncoding.Column.addU(builder, tarr);
+    return NetEncoding.Column.endColumn(builder);
+  });
+
+  const encColumns = NetEncoding.Matrix.createColumnsVector(builder, cols);
+
+  let encColIndex;
+  if (colIndex) {
+    encColIndex = encodeTypedArray(
+      builder,
+      NetEncoding.TypedArray.JSONEncodedArray,
+      utf8Encoder.encode(JSON.stringify(colIndex))
+    );
+  }
+
+  NetEncoding.Matrix.startMatrix(builder);
+  NetEncoding.Matrix.addNRows(builder, columns[0].length);
+  NetEncoding.Matrix.addNCols(builder, columns.length);
+  NetEncoding.Matrix.addColumns(builder, encColumns);
+  if (colIndex) {
+    NetEncoding.Matrix.addColIndexType(
+      builder,
+      NetEncoding.TypedArray.JSONEncodedArray
+    );
+    NetEncoding.Matrix.addColIndex(builder, encColIndex);
+  }
+  const root = NetEncoding.Matrix.endMatrix(builder);
+  builder.finish(root);
+  return builder.asUint8Array();
+}
+
+const anAnnotationsObsFBSResponse = (() => {
+  const columns = _.zip(...anAnnotationsObsJSONResponse.data).slice(1);
+  return encodeMatrix(columns, anAnnotationsObsJSONResponse.names);
+})();
+
+const anAnnotationsVarFBSResponse = (() => {
+  const columns = _.zip(...anAnnotationsVarJSONResponse.data).slice(1);
+  return encodeMatrix(columns, anAnnotationsVarJSONResponse.names);
+})();
+
+const aLayoutFBSResponse = (() => {
+  const coords = [
+    new Float32Array(nObs).fill(Math.random()),
+    new Float32Array(nObs).fill(Math.random())
+  ];
+  const builder = new flatbuffers.Builder(1024);
+
+  const cols = _.map(coords, carr => {
+    const cdv = NetEncoding.Float32Array.createDataVector(builder, carr);
+    NetEncoding.Float32Array.startFloat32Array(builder);
+    NetEncoding.Float32Array.addData(builder, cdv);
+    const floatArr = NetEncoding.Float32Array.endFloat32Array(builder);
+
+    NetEncoding.Column.startColumn(builder);
+    NetEncoding.Column.addUType(builder, NetEncoding.TypedArray.Float32Array);
+    NetEncoding.Column.addU(builder, floatArr);
+    return NetEncoding.Column.endColumn(builder);
+  });
+
+  const columns = NetEncoding.Matrix.createColumnsVector(builder, cols);
+
+  NetEncoding.Matrix.startMatrix(builder);
+  NetEncoding.Matrix.addNRows(builder, nObs);
+  NetEncoding.Matrix.addNCols(builder, coords.length);
+  NetEncoding.Matrix.addColumns(builder, columns);
+  const matrix = NetEncoding.Matrix.endMatrix(builder);
+  builder.finish(matrix);
+  return builder.asUint8Array();
+})();
+
+const aDataObsResponse = {
+  var: [2, 4, 29],
+  obs: _()
+    .range(nObs)
+    .map(idx => [idx, Math.random(), Math.random(), Math.random()])
+    .value()
+};
+
+export {
+  aLayoutFBSResponse as layoutObs,
+  aDataObsResponse as dataObs,
+  anAnnotationsVarFBSResponse as annotationsVar,
+  anAnnotationsObsFBSResponse as annotationsObs,
+  aSchemaResponse as schema,
+  aConfigResponse as config
+};

client/__tests__/util/stateManager/universe.test.js

@@ -0,0 +1,66 @@
+import * as Universe from "../../../src/util/stateManager/universe";
+import * as Dataframe from "../../../src/util/dataframe";
+import * as REST from "./sampleResponses";
+
+describe("createUniverseFromResponse", () => {
+  /*
+  test createUniverseFromResponse - this function converts
+  a set of REST 0.2 responses into a "new" Universe.
+
+  createUniverseFromResponse(
+    configResponse,
+    schemaResponse,
+    annotationsObsResponse,
+    annotationsVarResponse,
+    layoutObsResponse
+  ) --> Universe
+
+  where:
+  configResponse: GET /.../config
+  schemaResponse: GET /.../schema
+  annotationsObsResponse: GET /.../annotations/obs
+  annotationsVarResponse: GET /.../annotations/var
+  layoutObsResponse: GET /.../layout/obs
+
+  See spec in docs/REST_API.md.
+  */
+
+  test("create from test data", () => {
+    /*
+    create a universe from sample data nad validate its shape & contents
+    */
+    const { nObs, nVar } = REST.schema.schema.dataframe;
+    const universe = Universe.createUniverseFromResponse(
+      REST.config,
+      REST.schema,
+      REST.annotationsObs,
+      REST.annotationsVar,
+      REST.layoutObs
+    );
+
+    expect(universe).toBeDefined();
+    expect(universe).toMatchObject(
+      expect.objectContaining({
+        nObs,
+        nVar,
+        schema: REST.schema.schema,
+        obsAnnotations: expect.any(Dataframe.Dataframe),
+        varAnnotations: expect.any(Dataframe.Dataframe),
+        obsLayout: expect.any(Dataframe.Dataframe),
+        varData: expect.any(Dataframe.Dataframe)
+      })
+    );
+
+    expect(universe.obsAnnotations.dims).toEqual([
+      nObs,
+      REST.schema.schema.annotations.obs.length
+    ]);
+    expect(universe.obsLayout.dims).toEqual([nObs, 2]);
+    expect(universe.obsLayout.colIndex.keys()).toEqual(["X", "Y"]);
+    expect(universe.varAnnotations.dims).toEqual([
+      nVar,
+      REST.schema.schema.annotations.var.length
+    ]);
+    expect(universe.varData.isEmpty()).toBeTruthy();
+  });
+});

client/__tests__/util/stateManager/world.test.js

@@ -0,0 +1,180 @@
+import _ from "lodash";
+import * as Universe from "../../../src/util/stateManager/universe";
+import * as World from "../../../src/util/stateManager/world";
+import * as Dataframe from "../../../src/util/dataframe";
+import Crossfilter from "../../../src/util/typedCrossfilter";
+import * as REST from "./sampleResponses";
+import {
+  obsAnnoDimensionName,
+  layoutDimensionName
+} from "../../../src/util/nameCreators";
+
+/*
+Helper - creates universe, world, corssfilter and dimensionMap from
+the default REST test response.
+*/
+const defaultBigBang = () => {
+  /* create unverse, world, crossfilter and dimensionMap */
+  /* create universe */
+  const universe = Universe.createUniverseFromResponse(
+    REST.config,
+    REST.schema,
+    REST.annotationsObs,
+    REST.annotationsVar,
+    REST.layoutObs
+  );
+  /* create world */
+  const world = World.createWorldFromEntireUniverse(universe);
+  /* create crossfilter */
+  const crossfilter = Crossfilter(world.obsAnnotations);
+  /* create dimension map */
+  const dimensionMap = World.createObsDimensionMap(crossfilter, world);
+
+  return {
+    universe,
+    world,
+    crossfilter,
+    dimensionMap
+  };
+};
+
+describe("createWorldFromEntireUniverse", () => {
+  test("create from REST sample", () => {
+    const universe = Universe.createUniverseFromResponse(
+      REST.config,
+      REST.schema,
+      REST.annotationsObs,
+      REST.annotationsVar,
+      REST.layoutObs
+    );
+    expect(universe).toBeDefined();
+
+    const world = World.createWorldFromEntireUniverse(universe);
+    expect(world).toBeDefined();
+
+    expect(world).toMatchObject(
+      expect.objectContaining({
+        nObs: universe.nObs,
+        nVar: universe.nVar,
+        schema: universe.schema,
+        obsAnnotations: universe.obsAnnotations,
+        varAnnotations: universe.varAnnotations,
+        obsLayout: universe.obsLayout,
+        varData: expect.any(Dataframe.Dataframe)
+      })
+    );
+  });
+});
+
+describe("createWorldFromCurrentSelection", () => {
+  test("create from REST sample", () => {
+    const {
+      universe,
+      world: originalWorld,
+      crossfilter,
+      dimensionMap
+    } = defaultBigBang();
+
+    /* mock a selection */
+    dimensionMap[obsAnnoDimensionName("field1")].filterRange([0, 5]);
+    dimensionMap[obsAnnoDimensionName("field3")].filterExact(false);
+
+    /* create the world from the selection */
+    const world = World.createWorldFromCurrentSelection(
+      universe,
+      originalWorld,
+      crossfilter
+    );
+    expect(world).toBeDefined();
+    expect(world.nObs).toEqual(crossfilter.countFiltered());
+
+    /*
+    calculate expected values and match against result
+    */
+
+    /* matchFilter must match the dimension filters above */
+    const matchFilter = (df, row) => {
+      const field1 = df.at(row, "field1");
+      const field3 = df.at(row, "field3");
+      return field1 >= 0 && field1 < 5 && !field3;
+    };
+    const matchingIndices = _()
+      .range(universe.nObs)
+      .filter(idx => matchFilter(universe.obsAnnotations, idx))
+      .value();
+
+    expect(world).toMatchObject(
+      expect.objectContaining({
+        nObs: matchingIndices.length,
+        nVar: universe.nVar,
+        schema: universe.schema,
+        obsAnnotations: expect.any(Dataframe.Dataframe),
+        varAnnotations: universe.varAnnotations,
+        obsLayout: expect.any(Dataframe.Dataframe),
+        varData: expect.any(Dataframe.Dataframe)
+      })
+    );
+
+    expect(world.obsAnnotations.rowIndex.keys()).toEqual(
+      new Int32Array(matchingIndices)
+    );
+    expect(world.obsAnnotations.colIndex.keys()).toEqual(
+      universe.obsAnnotations.colIndex.keys()
+    );
+    expect(world.obsLayout.rowIndex.keys()).toEqual(
+      new Int32Array(matchingIndices)
+    );
+    expect(world.obsLayout.colIndex.keys()).toEqual(["X", "Y"]);
+  });
+});
+
+describe("createObsDimensionMap", () => {
+  test("when universe eq world", () => {
+    /*
+    check for:
+    - creates a dimension for all obsAnnotations, PLUS X/Y layout
+    - check that dimension typing is sane
+    */
+
+    const { dimensionMap } = defaultBigBang();
+    const annotationNames = _.map(
+      REST.schema.schema.annotations.obs,
+      c => c.name
+    );
+    const schemaByObsName = _.keyBy(REST.schema.schema.annotations.obs, "name");
+    expect(dimensionMap).toBeDefined();
+    annotationNames.forEach(name => {
+      const dim = dimensionMap[obsAnnoDimensionName(name)];
+      if (name === "name") {
+        expect(dim).toBeUndefined();
+      } else {
+        const { type } = schemaByObsName[name];
+        if (type === "string" || type === "boolean" || type === "categorical") {
+          expect(dim).toBeInstanceOf(Crossfilter.EnumDimension);
+        } else {
+          expect(dim).toBeInstanceOf(Crossfilter.ScalarDimension);
+        }
+      }
+    });
+    expect(dimensionMap[layoutDimensionName("XY")]).toBeInstanceOf(
+      Crossfilter.SpatialDimension
+    );
+  });
+});
+
+describe("createVarDataDimension", () => {
+  /* create default universe */
+  const { world, crossfilter } = defaultBigBang();
+  world.varData = world.varData.withCol(
+    "GENE",
+    Float32Array.from(_.range(world.nObs))
+  );
+  const result = World.createVarDataDimension(world, crossfilter, "GENE");
+  expect(result).toBeInstanceOf(Crossfilter.ScalarDimension);
+});
+
+describe("worldEqUniverse", () => {
+  const { universe, world } = defaultBigBang();
+  const result = World.worldEqUniverse(world, universe);
+  expect(result).toBe(true);
+});

client/__tests__/util/stateManager/worldUtil.test.js

@@ -0,0 +1,62 @@
+import {
+  countCategoryValues2D,
+  clearCaches
+} from "../../../src/util/stateManager/worldUtil";
+import * as Dataframe from "../../../src/util/dataframe";
+
+describe("WorldUtil cache management", () => {
+  test("empty", () => {
+    const count = countCategoryValues2D(
+      "a",
+      "b",
+      new Dataframe.Dataframe([0, 0], [])
+    );
+    expect(count).toMatchObject(new Map());
+    expect(count.size).toBe(0);
+  });
+
+  test("simple couts", () => {
+    const df = new Dataframe.Dataframe(
+      [3, 2],
+      [[0, 0, 1], [false, true, false]],
+      null,
+      new Dataframe.KeyIndex(["a", "b"])
+    );
+    const count = countCategoryValues2D("a", "b", df);
+    expect(count).toMatchObject(
+      new Map([
+        [0, new Map([[true, 1], [false, 1]])],
+        [1, new Map([[false, 1]])]
+      ])
+    );
+  });
+
+  test("memo cache clear", () => {
+    clearCaches();
+    const df1 = new Dataframe.Dataframe([0, 0], []);
+    const df2 = new Dataframe.Dataframe(
+      [3, 2],
+      [[0, 0, 1], [false, true, false]],
+      null,
+      new Dataframe.KeyIndex(["a", "b"])
+    );
+
+    const count1 = countCategoryValues2D("a", "b", df1);
+    const count2 = countCategoryValues2D("a", "b", df1);
+    const count3 = countCategoryValues2D("a", "b", df1.clone());
+    const count4 = countCategoryValues2D("a", "b", df2);
+
+    clearCaches();
+    const count10 = countCategoryValues2D("a", "b", df1);
+    const count11 = countCategoryValues2D("a", "b", df2);
+
+    expect(count1).toEqual(count2);
+    expect(count1).toEqual(count3);
+    expect(count1).toEqual(count10);
+    expect(count1).not.toBe(count3);
+    expect(count1).not.toBe(count10);
+
+    expect(count4).toEqual(count11);
+    expect(count4).not.toBe(count11);
+  });
+});

client/__tests__/util/typedCrossfilter/bitArray.test.js

@@ -0,0 +1,185 @@
+// jshint esversion: 6
+
+import BitArray from "../../../src/util/typedCrossfilter/bitArray";
+
+const defaultTestLength = 8;
+
+describe("default select state", () => {
+  test("newly created Bitarray should be deselected", () => {
+    const ba = new BitArray(defaultTestLength);
+    expect(ba).toBeDefined();
+
+    for (let i = 0; i < defaultTestLength; i += 1) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+
+    const dim = ba.allocDimension();
+    expect(dim).toBeDefined();
+    for (let i = 0; i < defaultTestLength; i += 1) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+
+    ba.freeDimension(dim);
+    for (let i = 0; i < defaultTestLength; i += 1) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+  });
+});
+
+describe("select and deselect", () => {
+  test("selectAll and deselectAll", () => {
+    const ba = new BitArray(defaultTestLength);
+    expect(ba).toBeDefined();
+    const dim1 = ba.allocDimension();
+    expect(dim1).toBeDefined();
+    ba.selectAll(dim1);
+
+    for (let i = 0; i < defaultTestLength; i += 1) {
+      expect(ba.isSelected(i)).toEqual(true);
+    }
+
+    const dim2 = ba.allocDimension();
+    expect(dim2).toBeDefined();
+    for (let i = 0; i < defaultTestLength; i += 1) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+
+    ba.selectAll(dim2);
+    for (let i = 0; i < defaultTestLength; i += 1) {
+      expect(ba.isSelected(i)).toEqual(true);
+    }
+
+    ba.deselectAll(dim1);
+    for (let i = 0; i < defaultTestLength; i += 1) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+
+    ba.deselectAll(dim2);
+    for (let i = 0; i < defaultTestLength; i += 1) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+
+    ba.selectAll(dim1);
+    ba.selectAll(dim2);
+    for (let i = 0; i < defaultTestLength; i += 1) {
+      expect(ba.isSelected(i)).toEqual(true);
+    }
+
+    ba.freeDimension(dim1);
+    for (let i = 0; i < defaultTestLength; i += 1) {
+      expect(ba.isSelected(i)).toEqual(true);
+    }
+
+    ba.freeDimension(dim2);
+    for (let i = 0; i < defaultTestLength; i += 1) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+  });
+
+  test("selectOne and deselectOne", () => {
+    const ba = new BitArray(defaultTestLength);
+    expect(ba).toBeDefined();
+    const dim = ba.allocDimension();
+    expect(dim).toBeDefined();
+
+    ba.selectOne(dim, 0);
+    expect(ba.isSelected(0)).toEqual(true);
+    for (let i = 1; i < defaultTestLength; i += 1) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+
+    ba.deselectOne(dim, 0);
+    for (let i = 0; i < defaultTestLength; i += 1) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+
+    ba.selectOne(dim, 1);
+    expect(ba.isSelected(1)).toEqual(true);
+    expect(ba.isSelected(0)).toEqual(false);
+    for (let i = 2; i < defaultTestLength; i += 1) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+
+    ba.selectAll(dim);
+    ba.deselectOne(dim, defaultTestLength - 1);
+    expect(ba.isSelected(defaultTestLength - 1)).toEqual(false);
+    for (let i = 0; i < defaultTestLength - 1; i += 1) {
+      expect(ba.isSelected(i)).toEqual(true);
+    }
+  });
+});
+
+describe("selectionCount", () => {
+  test("simple", () => {
+    const ba = new BitArray(defaultTestLength);
+    expect(ba).toBeDefined();
+    const dim1 = ba.allocDimension();
+    expect(dim1).toBeDefined();
+    const dim2 = ba.allocDimension();
+    expect(dim2).toBeDefined();
+
+    expect(ba.selectionCount()).toEqual(0);
+    ba.selectAll(dim1);
+    expect(ba.selectionCount()).toEqual(0);
+    ba.selectAll(dim2);
+    expect(ba.selectionCount()).toEqual(defaultTestLength);
+
+    for (let i = 0; i < defaultTestLength; i += 1) {
+      ba.deselectOne(dim1, i);
+      expect(ba.selectionCount()).toEqual(defaultTestLength - i - 1);
+      expect(ba.selectionCount()).toEqual(ba.countAllOnes());
+    }
+
+    ba.freeDimension(dim1);
+    ba.freeDimension(dim2);
+  });
+});
+
+describe("fillBySelection", () => {
+  test("sets values correctly", () => {
+    const ba = new BitArray(defaultTestLength);
+    expect(ba).toBeDefined();
+    const dim = ba.allocDimension();
+    expect(dim).toBeDefined();
+
+    const arr = new Int32Array(defaultTestLength);
+    arr.fill(0);
+    const truth = new Int32Array(defaultTestLength);
+    truth.fill(0);
+
+    // initial state should be deselected
+    ba.fillBySelection(arr, 1, 0);
+    expect(arr).toEqual(expect.not.arrayContaining([1]));
+
+    // selectAll
+    ba.selectAll(dim);
+    ba.fillBySelection(arr, 1, 0);
+    expect(arr).toEqual(expect.not.arrayContaining([0]));
+
+    // deselectOne
+    ba.deselectOne(dim, 3);
+    ba.fillBySelection(arr, 1, 0);
+    truth.fill(1);
+    truth[3] = 0;
+    expect(arr).toEqual(truth);
+
+    // deselectAll
+    ba.deselectAll(dim);
+    ba.fillBySelection(arr, 1, 0);
+    truth.fill(0);
+    expect(arr).toEqual(truth);
+
+    // selectOne
+    ba.selectOne(dim, 5);
+    ba.fillBySelection(arr, 6, 1);
+    truth.fill(1);
+    truth[5] = 6;
+    expect(arr).toEqual(truth);
+
+    // should be deselected after dimension disposal
+    ba.freeDimension(dim);
+    ba.fillBySelection(arr, 3, 9);
+    truth.fill(9);
+    expect(arr).toEqual(truth);
+  });
+});

client/__tests__/util/typedCrossfilter/positiveInterval.test.js

@@ -0,0 +1,141 @@
+// jshint esversion: 6
+
+// const PositiveIntervals = require("../../src/util/typedCrossfilter/positiveIntervals");
+import PositiveIntervals from "../../../src/util/typedCrossfilter/positiveIntervals";
+
+describe("canonicalize", () => {
+  test("empty", () => {
+    expect(PositiveIntervals.canonicalize([])).toEqual([]);
+  });
+
+  test("simple, already correct", () => {
+    expect(PositiveIntervals.canonicalize([[0, 1]])).toEqual([[0, 1]]);
+    expect(PositiveIntervals.canonicalize([[0, 1], [2, 3]])).toEqual([
+      [0, 1],
+      [2, 3]
+    ]);
+  });
+
+  test("non-canonical, need to be canonicalized", () => {
+    expect(PositiveIntervals.canonicalize([[0, 1], [1, 2]])).toEqual([[0, 2]]);
+    expect(PositiveIntervals.canonicalize([[1, 2], [2, 3]])).toEqual([[1, 3]]);
+  });
+});
+
+describe("union", () => {
+  test("empty range", () => {
+    expect(PositiveIntervals.union([], [])).toEqual([]);
+    expect(PositiveIntervals.union([], [[1, 2]])).toEqual([[1, 2]]);
+    expect(PositiveIntervals.union([], [[1, 2], [3, 4]])).toEqual([
+      [1, 2],
+      [3, 4]
+    ]);
+    expect(PositiveIntervals.union([[3, 4]], [])).toEqual([[3, 4]]);
+    expect(PositiveIntervals.union([[1, 2], [3, 4]], [])).toEqual([
+      [1, 2],
+      [3, 4]
+    ]);
+    expect(PositiveIntervals.union([[3, 3]], [])).toEqual([[3, 3]]);
+    expect(PositiveIntervals.union([], [[3, 3]])).toEqual([[3, 3]]);
+  });
+
+  test("simple ranges", () => {
+    expect(PositiveIntervals.union([[1, 2]], [[2, 3]])).toEqual([[1, 3]]);
+    expect(PositiveIntervals.union([[2, 3]], [[1, 2]])).toEqual([[1, 3]]);
+    expect(PositiveIntervals.union([[1, 2]], [[3, 4]])).toEqual([
+      [1, 2],
+      [3, 4]
+    ]);
+    expect(
+      PositiveIntervals.union([[1, 2], [3, 4]], [[6, 7], [19, 40]])
+    ).toEqual([[1, 2], [3, 4], [6, 7], [19, 40]]);
+    expect(PositiveIntervals.union([[1, 4]], [[1, 1], [3, 4]])).toEqual([
+      [1, 4]
+    ]);
+    expect(PositiveIntervals.union([[3, 3]], [[4, 4]])).toEqual([
+      [3, 3],
+      [4, 4]
+    ]);
+  });
+});
+
+describe("intersection", () => {
+  test("empty range", () => {
+    expect(PositiveIntervals.intersection([], [])).toEqual([]);
+    expect(PositiveIntervals.intersection([], [[1, 2]])).toEqual([]);
+    expect(PositiveIntervals.intersection([[1, 2]], [])).toEqual([]);
+  });
+
+  test("simple", () => {
+    expect(PositiveIntervals.intersection([[1, 2]], [[2, 3]])).toEqual([]);
+    expect(PositiveIntervals.intersection([[2, 3]], [[1, 2]])).toEqual([]);
+    expect(PositiveIntervals.intersection([[1, 10]], [[1, 10]])).toEqual([
+      [1, 10]
+    ]);
+    expect(PositiveIntervals.intersection([[1, 10]], [[2, 8]])).toEqual([
+      [2, 8]
+    ]);
+    expect(PositiveIntervals.intersection([[2, 8]], [[1, 10]])).toEqual([
+      [2, 8]
+    ]);
+    expect(PositiveIntervals.intersection([[1, 10]], [[2, 12]])).toEqual([
+      [2, 10]
+    ]);
+    expect(PositiveIntervals.intersection([[2, 12]], [[1, 10]])).toEqual([
+      [2, 10]
+    ]);
+    expect(PositiveIntervals.intersection([[1, 10]], [[1, 8]])).toEqual([
+      [1, 8]
+    ]);
+    expect(PositiveIntervals.intersection([[1, 8]], [[1, 10]])).toEqual([
+      [1, 8]
+    ]);
+    expect(PositiveIntervals.intersection([[1, 10]], [[1, 2], [6, 9]])).toEqual(
+      [[1, 2], [6, 9]]
+    );
+    expect(PositiveIntervals.intersection([[0, 2638]], [[1363, 2638]])).toEqual(
+      [[1363, 2638]]
+    );
+    expect(PositiveIntervals.intersection([[1, 2]], [[1, 2]])).toEqual([
+      [1, 2]
+    ]);
+  });
+});
+
+describe("difference", () => {
+  test("empty", () => {
+    expect(PositiveIntervals.difference([], [])).toEqual([]);
+    expect(PositiveIntervals.difference([], [[1, 10]])).toEqual([]);
+    expect(PositiveIntervals.difference([[1, 10]], [])).toEqual([[1, 10]]);
+  });
+
+  test("simple", () => {
+    expect(PositiveIntervals.difference([[1, 2], [3, 4]], [])).toEqual([
+      [1, 2],
+      [3, 4]
+    ]);
+    expect(PositiveIntervals.difference([[1, 2], [3, 10]], [[5, 10]])).toEqual([
+      [1, 2],
+      [3, 5]
+    ]);
+    expect(PositiveIntervals.difference([[1, 2], [3, 10]], [[0, 5]])).toEqual([
+      [5, 10]
+    ]);
+    expect(
+      PositiveIntervals.difference([[0, 2638]], [[0, 1363], [2055, 2638]])
+    ).toEqual([[1363, 2055]]);
+    expect(
+      PositiveIntervals.difference([[0, 1363], [2055, 2638]], [[0, 2638]])
+    ).toEqual([]);
+    expect(PositiveIntervals.difference([[0, 10]], [[0, 1]])).toEqual([
+      [1, 10]
+    ]);
+    expect(PositiveIntervals.difference([[0, 10]], [[1, 2]])).toEqual([
+      [0, 1],
+      [2, 10]
+    ]);
+    expect(PositiveIntervals.difference([[0, 10]], [[9, 10]])).toEqual([
+      [0, 9]
+    ]);
+  });
+});

client/__tests__/util/typedCrossfilter/sort.test.js

@@ -0,0 +1,56 @@
+import { sort, sortIndex } from "../../../src/util/typedCrossfilter/sort";
+
+function fillRange(arr, start = 0) {
+  const larr = arr;
+  for (let i = 0, len = larr.length; i < len; i += 1) {
+    larr[i] = i + start;
+  }
+  return larr;
+}
+
+function fillRand(arr) {
+  for (let i = 0, len = arr.length; i < len; i += 1) {
+    arr[i] = Math.random();
+  }
+  return arr;
+}
+
+describe("sort", () => {
+  [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map(Type =>
+    test(Type.name, () => {
+      expect(sort(Type.from([6, 5, 4, 3, 2, 1, 0]))).toMatchObject(
+        Type.from([0, 1, 2, 3, 4, 5, 6])
+      );
+      expect(sort(Type.from([6, 5, 4, 3, 2, 1]))).toMatchObject(
+        Type.from([1, 2, 3, 4, 5, 6])
+      );
+
+      const source = fillRand(new Type(1000));
+      expect(sort(Type.from(source))).toMatchObject(Type.from(source).sort());
+    })
+  );
+});
+
+describe("sortIndex", () => {
+  [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map(Type =>
+    test(Type.name, () => {
+      const source1 = Type.from([6, 5, 4, 3, 2, 1, 0]);
+      const index1 = fillRange(new Uint32Array(source1.length));
+      expect(sortIndex(index1, source1)).toMatchObject(
+        index1.sort((a, b) => source1[a] - source1[b])
+      );
+
+      const source2 = Type.from([6, 5, 4, 3, 2, 1]);
+      const index2 = fillRange(new Uint32Array(source2.length));
+      expect(sortIndex(index2, source2)).toMatchObject(
+        index2.sort((a, b) => source1[a] - source1[b])
+      );
+
+      const source3 = fillRand(new Type(1000));
+      const index3 = fillRange(new Uint32Array(source3.length));
+      expect(sortIndex(index3, source3)).toMatchObject(
+        index3.sort((a, b) => source1[a] - source1[b])
+      );
+    })
+  );
+});

client/__tests__/util/typedCrossfilter/typedCrossfilter.test.js

@@ -0,0 +1,590 @@
+// jshint esversion: 6
+import _ from "lodash";
+import crossfilter from "../../../src/util/typedCrossfilter";
+
+const someData = [
+  {
+    date: "2011-11-14T16:17:54Z",
+    quantity: 2,
+    total: 190,
+    tip: 100,
+    type: "tab",
+    productIDs: ["001"]
+  },
+  {
+    date: "2011-11-14T16:20:19Z",
+    quantity: 2,
+    total: 190,
+    tip: 100,
+    type: "tab",
+    productIDs: ["001", "005"]
+  },
+  {
+    date: "2011-11-14T16:28:54Z",
+    quantity: 1,
+    total: 300,
+    tip: 200,
+    type: "visa",
+    productIDs: ["004", "005"]
+  },
+  {
+    date: "2011-11-14T16:30:43Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["001", "002"]
+  },
+  {
+    date: "2011-11-14T16:48:46Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["005"]
+  },
+  {
+    date: "2011-11-14T16:53:41Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["001", "004", "005"]
+  },
+  {
+    date: "2011-11-14T16:54:06Z",
+    quantity: 1,
+    total: 100,
+    tip: 0,
+    type: "cash",
+    productIDs: ["001", "002", "003", "004", "005"]
+  },
+  {
+    date: "2011-11-14T16:58:03Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["001"]
+  },
+  {
+    date: "2011-11-14T17:07:21Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["004", "005"]
+  },
+  {
+    date: "2011-11-14T17:22:59Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["001", "002", "004", "005"]
+  },
+  {
+    date: "2011-11-14T17:25:45Z",
+    quantity: 2,
+    total: 200,
+    tip: 0,
+    type: "cash",
+    productIDs: ["002"]
+  },
+  {
+    date: "2011-11-14T17:29:52Z",
+    quantity: 1,
+    total: 200,
+    tip: 100,
+    type: "visa",
+    productIDs: ["004"]
+  }
+];
+
+function groupReduce(data, valueMap, valueReduce, valueInit) {
+  return _.reduce(
+    data,
+    (acc, value) => {
+      const k = valueMap(value);
+      let r = _.find(acc, o => o.key === k);
+      if (!r) {
+        r = { key: k, value: valueInit() };
+        acc.push(r);
+      }
+      r.value = valueReduce(r.value, value);
+      return acc;
+    },
+    []
+  ).sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
+}
+
+function groupCount(data, map) {
+  return groupReduce(data, map, p => p + 1, () => 0);
+}
+
+function groupSum(data, map) {
+  return groupReduce(
+    data,
+    map,
+    (p, v) => {
+      p += map(v);
+      return p;
+    },
+    () => 0
+  );
+}
+
+let payments = null;
+beforeEach(() => {
+  payments = crossfilter(someData);
+});
+
+describe("typedCrossfilter", () => {
+  test("alloc and free", () => {
+    expect(payments).toBeDefined();
+    expect(payments.size()).toEqual(someData.length);
+    expect(payments.all()).toEqual(someData);
+
+    const quantity = payments.dimension(
+      crossfilter.ScalarDimension,
+      (i, data) => data[i].quantity,
+      Int32Array
+    );
+    expect(quantity).toBeDefined();
+    expect(quantity.id()).toBeDefined();
+
+    quantity.dispose();
+    expect(payments.size()).toEqual(someData.length);
+    expect(payments.all()).toEqual(someData);
+  });
+
+  test("filterAll and filterNone", () => {
+    expect(payments).toBeDefined();
+    const quantity = payments.dimension(
+      crossfilter.ScalarDimension,
+      (i, data) => data[i].quantity,
+      Int32Array
+    );
+    const tip = payments.dimension(
+      crossfilter.ScalarDimension,
+      (i, data) => data[i].tip,
+      Float32Array
+    );
+    const total = payments.dimension(
+      crossfilter.ScalarDimension,
+      (i, data) => data[i].total,
+      Float32Array
+    );
+    const type = payments.dimension(
+      crossfilter.EnumDimension,
+      (i, data) => data[i].type
+    );
+
+    expect(quantity).toBeDefined();
+    expect(tip).toBeDefined();
+    expect(total).toBeDefined();
+    expect(type).toBeDefined();
+
+    // initially, all should be filtered
+    expect(payments.allFiltered()).toHaveLength(payments.size());
+    expect(payments.allFiltered()).toEqual(payments.all());
+    expect(payments.countFiltered()).toEqual(someData.length);
+
+    // filterAll
+    tip.filterAll(); // should change nothing
+    expect(payments.allFiltered()).toEqual(payments.all());
+    expect(payments.countFiltered()).toEqual(someData.length);
+
+    // ditto
+    total.filterAll();
+    expect(payments.allFiltered()).toEqual(payments.all());
+    expect(payments.countFiltered()).toEqual(someData.length);
+
+    // filterNone
+    type.filterNone();
+    expect(payments.allFiltered()).toEqual([]);
+    expect(payments.countFiltered()).toEqual(0);
+
+    quantity.filterNone();
+    expect(payments.allFiltered()).toEqual([]);
+    expect(payments.countFiltered()).toEqual(0);
+
+    // invert the first none; should have no effect because type is
+    // still not filtered
+    quantity.filterAll();
+    expect(payments.allFiltered()).toEqual([]);
+    expect(payments.countFiltered()).toEqual(0);
+
+    // filter all of type; should select all
+    type.filterAll();
+    expect(payments.allFiltered()).toEqual(payments.all());
+    expect(payments.countFiltered()).toEqual(payments.size());
+  });
+
+  test("filterExact", () => {
+    expect(payments).toBeDefined();
+    const quantity = payments.dimension(
+      crossfilter.ScalarDimension,
+      (i, data) => data[i].quantity,
+      Int32Array
+    );
+    const tip = payments.dimension(
+      crossfilter.ScalarDimension,
+      (i, data) => data[i].tip,
+      Float32Array
+    );
+    const type = payments.dimension(
+      crossfilter.EnumDimension,
+      (i, data) => data[i].type
+    );
+
+    quantity.filterExact(1);
+    expect(payments.countFiltered()).toEqual(
+      _.countBy(someData, "quantity")[1]
+    );
+    expect(payments.allFiltered()).toEqual(_.filter(someData, { quantity: 1 }));
+
+    tip.filterExact(0);
+    expect(payments.allFiltered()).toEqual(
+      _.filter(someData, { tip: 0, quantity: 1 })
+    );
+
+    type.filterExact("cash");
+    expect(payments.allFiltered()).toEqual(
+      _.filter(someData, { tip: 0, quantity: 1, type: "cash" })
+    );
+  });
+
+  test("filterRange", () => {
+    expect(payments).toBeDefined();
+    const quantity = payments.dimension(
+      crossfilter.ScalarDimension,
+      (i, data) => data[i].quantity,
+      Int32Array
+    );
+    const tip = payments.dimension(
+      crossfilter.ScalarDimension,
+      (i, data) => data[i].tip,
+      Float32Array
+    );
+    const total = payments.dimension(
+      crossfilter.ScalarDimension,
+      (i, data) => data[i].total,
+      Float32Array
+    );
+    const type = payments.dimension(
+      crossfilter.EnumDimension,
+      (i, data) => data[i].type
+    );
+
+    tip.filterRange([0, 91]);
+    expect(payments.allFiltered()).toEqual(
+      _(someData)
+        .filter(r => r.tip >= 0 && r.tip < 91)
+        .value()
+    );
+
+    tip.filterRange([0, 90]);
+    expect(payments.allFiltered()).toEqual(
+      _(someData)
+        .filter(r => r.tip >= 0 && r.tip < 90)
+        .value()
+    );
+
+    tip.filterRange([1, 90]);
+    expect(payments.allFiltered()).toEqual(
+      _(someData)
+        .filter(r => r.tip >= 1 && r.tip < 91)
+        .value()
+    );
+  });
+
+  test("filterEnum", () => {
+    expect(payments).toBeDefined();
+    const quantity = payments.dimension(
+      crossfilter.ScalarDimension,
+      (i, data) => data[i].quantity,
+      Int32Array
+    );
+    const tip = payments.dimension(
+      crossfilter.ScalarDimension,
+      (i, data) => data[i].tip,
+      Float32Array
+    );
+    const total = payments.dimension(
+      crossfilter.ScalarDimension,
+      (i, data) => data[i].total,
+      Float32Array
+    );
+    const type = payments.dimension(
+      crossfilter.EnumDimension,
+      (i, data) => data[i].type
+    );
+
+    type.filterEnum(["tab", "cash"]);
+    expect(payments.allFiltered()).toEqual(
+      _(someData)
+        .filter(r => r.type === "cash" || r.type === "tab")
+        .value()
+    );
+
+    tip.filterEnum([0, 100]);
+    expect(payments.allFiltered()).toEqual(
+      _(someData)
+        .filter(r => r.type === "cash" || r.type === "tab")
+        .filter(r => r.tip === 0 || r.tip === 100)
+        .value()
+    );
+  });
+
+  test("more than 32 dimensions", () => {
+    expect(payments).toBeDefined();
+    const quantity = payments.dimension(
+      crossfilter.ScalarDimension,
+      (i, data) => data[i].quantity,
+      Int32Array
+    );
+    const tip = payments.dimension(
+      crossfilter.ScalarDimension,
+      (i, data) => data[i].tip,
+      Float32Array
+    );
+    const total = payments.dimension(
+      crossfilter.ScalarDimension,
+      (i, data) => data[i].total,
+      Float32Array
+    );
+    const type = payments.dimension(
+      crossfilter.EnumDimension,
+      (i, data) => data[i].type
+    );
+
+    // Create a bunch of fake dimensions to ensure we can handle > 32
+    let dimMap = {};
+    for (let i = 0; i < 65; i++) {
+      dimMap[i] = payments.dimension(
+        crossfilter.ScalarDimension,
+        () => Math.random(),
+        Float32Array
+      );
+      expect(dimMap[i]).toBeDefined();
+      expect(dimMap[i].id()).toBeDefined();
+    }
+
+    // everything should start as selected/filtered
+    expect(payments.countFiltered()).toEqual(someData.length);
+
+    dimMap[0].filterAll();
+    dimMap[64].filterAll();
+    expect(payments.countFiltered()).toEqual(someData.length);
+
+    dimMap[33].filterNone();
+    expect(payments.allFiltered()).toEqual([]);
+
+    dimMap[33].filterAll();
+    expect(payments.allFiltered()).toEqual(someData);
+  });
+
+  test("group, default mapping, default reducer, no filter", () => {
+    expect(payments).toBeDefined();
+
+    const quantity = payments.dimension(
+      crossfilter.ScalarDimension,
+      (i, data) => data[i].quantity,
+      Int32Array
+    );
+    const tip = payments.dimension(
+      crossfilter.ScalarDimension,
+      (i, data) => data[i].tip,
+      Int32Array
+    );
+    const type = payments.dimension(
+      crossfilter.EnumDimension,
+      (i, data) => data[i].type
+    );
+    const total = payments.dimension(
+      crossfilter.ScalarDimension,
+      (i, data) => data[i].total,
+      Int32Array
+    );
+
+    _.each(
+      {
+        tip: tip.group(r => r),
+        type: type.group(),
+        total: total.group(),
+        quantity: quantity.group()
+      },
+      (grp, k) => {
+        const whatWeExpect = groupCount(someData, v => v[k]);
+        expect(grp.all()).toEqual(whatWeExpect);
+        expect(grp.size()).toEqual(whatWeExpect.length);
+        expect(grp.dispose()).toEqual(grp);
+      }
+    );
+  });
+
+  test("group, custom map, default reducer, no filters", () => {
+    expect(payments).toBeDefined();
+
+    // custom mapping in groups only works for scalar types.  Enums do not
+    // currently implement it.
+
+    const tip = payments.dimension(
+      crossfilter.ScalarDimension,
+      (i, data) => data[i].tip,
+      Int32Array
+    );
+    const totalX10 = payments.dimension(
+      crossfilter.ScalarDimension,
+      (i, data) => data[i].total * 10,
+      Int32Array
+    );
+    const type = payments.dimension(
+      crossfilter.EnumDimension,
+      (i, data) => data[i].type
+    );
+
+    const paymentsByTip_A = tip.group();
+    const paymentsByTip_B = tip.group(r => 10 * r);
+    const paymentsByType = type.group(); // identity only
+    const paymentsByTotalX10_A = totalX10.group();
+    const paymentsByTotalX10_B = totalX10.group(r => r / 10);
+
+    expect(paymentsByTip_A.all()).toEqual(groupCount(someData, v => v.tip));
+    expect(paymentsByTip_B.all()).toEqual(
+      groupCount(someData, v => 10 * v.tip)
+    );
+    expect(paymentsByType.all()).toEqual(groupCount(someData, v => v.type));
+    expect(paymentsByTotalX10_A.all()).toEqual(
+      groupCount(someData, v => 10 * v.total)
+    );
+    expect(paymentsByTotalX10_B.all()).toEqual(
+      groupCount(someData, v => (10 * v.total) / 10)
+    );
+
+    for (let i of [
+      paymentsByTip_A,
+      paymentsByTip_B,
+      paymentsByType,
+      paymentsByTotalX10_A,
+      paymentsByTotalX10_B,
+      tip,
+      totalX10,
+      type
+    ]) {
+      expect(i.dispose()).toEqual(i);
+    }
+  });
+
+  test("group, default map, custom reducer, no filters", () => {
+    expect(payments).toBeDefined();
+
+    const total = payments.dimension(
+      crossfilter.ScalarDimension,
+      (i, data) => data[i].total,
+      Float32Array
+    );
+    const type = payments.dimension(
+      crossfilter.EnumDimension,
+      (i, data) => data[i].type
+    );
+
+    const paymentsByTotal = total.group();
+    const paymentsByType = type.group();
+
+    // reduceCount
+    expect(paymentsByTotal.reduceCount()).toEqual(paymentsByTotal);
+    expect(paymentsByTotal.all()).toEqual(groupCount(someData, v => v.total));
+
+    // reduceSum
+    expect(paymentsByTotal.reduceSum(v => v.total)).toEqual(paymentsByTotal);
+    expect(paymentsByTotal.all()).toEqual(groupSum(someData, v => v.total));
+
+    // use custom reducers (my reducers) - count by three, init 1
+    expect(
+      paymentsByTotal.reduce((p, v) => (p += 3), (p, v) => (p -= 3), () => 1)
+    ).toEqual(paymentsByTotal);
+    expect(paymentsByTotal.all()).toEqual(
+      groupReduce(someData, v => v.total, (p, v) => p + 3, () => 1)
+    );
+
+    for (let i of [paymentsByTotal, paymentsByType, type]) {
+      expect(i.dispose()).toEqual(i);
+    }
+  });
+
+  test("group, default map, default reducer, filters", () => {
+    // From the docs:
+    // Note: a grouping intersects the crossfilter's current filters, except for the
+    // associated dimension's filter. Thus, group methods consider only records that
+    // satisfy every filter except this dimension's filter. So, if the crossfilter of
+    // payments is filtered by type and total, then group by total only observes the
+    // filter by type.
+
+    expect(payments).toBeDefined();
+
+    const tip = payments.dimension(
+      crossfilter.ScalarDimension,
+      (i, data) => data[i].tip,
+      Int32Array
+    );
+    const total = payments.dimension(
+      crossfilter.ScalarDimension,
+      (i, data) => data[i].total,
+      Int32Array
+    );
+    const type = payments.dimension(
+      crossfilter.EnumDimension,
+      (i, data) => data[i].type
+    );
+
+    const paymentsByTip = tip.group();
+    const paymentsByTotal = total.group();
+    const paymentsByType = type.group();
+
+    // 1. confirm that changing the filter on a dimension does NOT change that
+    // dimensions groups.
+    {
+      tip.filterAll(), total.filterAll(), type.filterAll();
+      let before = _.cloneDeep(paymentsByTip.all());
+      tip.filterExact(0);
+      expect(paymentsByTip.all()).toEqual(before);
+    }
+
+    // 2. confirm that changing a filter on a different dimension DOES change
+    // all other groups.
+    {
+      tip.filterAll(), total.filterAll(), type.filterAll();
+      const before = _.cloneDeep([paymentsByTotal.all(), paymentsByType.all()]);
+      tip.filterExact(0);
+      const after = [paymentsByTotal.all(), paymentsByType.all()];
+      expect(after).not.toEqual(before);
+      expect(after).toEqual([
+        groupReduce(
+          someData,
+          v => v.total,
+          (p, v) => (v.tip !== 0 ? p : p + 1),
+          () => 0
+        ),
+        groupReduce(
+          someData,
+          v => v.type,
+          (p, v) => (v.tip !== 0 ? p : p + 1),
+          () => 0
+        )
+      ]);
+    }
+
+    for (let i of [
+      paymentsByTip,
+      paymentsByTotal,
+      paymentsByType,
+      tip,
+      total,
+      type
+    ]) {
+      expect(i.dispose()).toEqual(i);
+    }
+  });
+});

client/__tests__/util/typedCrossfilter/util.test.js

@@ -0,0 +1,115 @@
+import {
+  fillRange,
+  sliceByIndex,
+  makeSortIndex,
+  lowerBound,
+  lowerBoundIndirect,
+  upperBoundIndirect
+} from "../../../src/util/typedCrossfilter/util";
+
+describe("fillRange", () => {
+  test("Array", () => {
+    expect(fillRange(new Array(6))).toMatchObject([0, 1, 2, 3, 4, 5]);
+    expect(fillRange(new Array(4), 1)).toMatchObject([1, 2, 3, 4]);
+    expect(fillRange([])).toMatchObject([]);
+  });
+
+  test("Uint32Array", () => {
+    expect(fillRange(new Uint32Array(6))).toMatchObject(
+      new Uint32Array([0, 1, 2, 3, 4, 5])
+    );
+    expect(fillRange(new Array(4), 1)).toMatchObject(
+      new Uint32Array([1, 2, 3, 4])
+    );
+    expect(fillRange([])).toMatchObject(new Uint32Array([]));
+  });
+});
+
+describe("sliceByIndex", () => {
+  test("Array", () => {
+    expect(sliceByIndex([0, 1, 2, 3, 4], [0, 1, 2])).toMatchObject([0, 1, 2]);
+    expect(sliceByIndex([0, 1, 2, 3, 4], [2, 1, 0])).toMatchObject([2, 1, 0]);
+    expect(sliceByIndex([0, 1, 2, 3, 4], [])).toMatchObject([]);
+    expect(sliceByIndex([], [])).toMatchObject([]);
+  });
+
+  test("Uint32Array", () => {
+    expect(
+      sliceByIndex([0, 1, 2, 3, 4], new Uint32Array([0, 1, 2]))
+    ).toMatchObject([0, 1, 2]);
+    expect(
+      sliceByIndex([0, 1, 2, 3, 4], new Uint32Array([2, 1, 0]))
+    ).toMatchObject([2, 1, 0]);
+    expect(sliceByIndex([0, 1, 2, 3, 4], new Uint32Array([]))).toMatchObject(
+      []
+    );
+    expect(sliceByIndex([], new Uint32Array([]))).toMatchObject([]);
+
+    expect(
+      sliceByIndex(new Uint32Array([0, 1, 2, 3, 4]), new Uint32Array([0, 1, 2]))
+    ).toMatchObject(new Uint32Array([0, 1, 2]));
+    expect(
+      sliceByIndex(new Uint32Array([0, 1, 2, 3, 4]), new Uint32Array([2, 1, 0]))
+    ).toMatchObject(new Uint32Array([2, 1, 0]));
+    expect(
+      sliceByIndex(
+        new Uint32Array([0, 1, 2, 3, 4]),
+        new Uint32Array(new Uint32Array([]))
+      )
+    ).toMatchObject(new Uint32Array([]));
+    expect(
+      sliceByIndex(new Uint32Array([]), new Uint32Array([]))
+    ).toMatchObject(new Uint32Array([]));
+  });
+
+  test("Float32Array", () => {
+    expect(
+      sliceByIndex(new Float32Array([0, 1, 2, 3, 4]), [0, 1, 2])
+    ).toMatchObject(new Float32Array([0, 1, 2]));
+    expect(
+      sliceByIndex(new Float32Array([0, 1, 2, 3, 4]), [2, 1, 0])
+    ).toMatchObject(new Float32Array([2, 1, 0]));
+    expect(sliceByIndex(new Float32Array([0, 1, 2, 3, 4]), [])).toMatchObject(
+      new Float32Array([])
+    );
+    expect(sliceByIndex(new Float32Array([]), [])).toMatchObject(
+      new Float32Array([])
+    );
+  });
+});
+
+describe("makeSortIndex", () => {
+  test("Array", () => {
+    expect(makeSortIndex([3, 2, 1, 0])).toMatchObject(
+      new Uint32Array([3, 2, 1, 0])
+    );
+    expect(makeSortIndex([3, 2, 1, 0, 4])).toMatchObject(
+      new Uint32Array([3, 2, 1, 0, 4])
+    );
+    expect(makeSortIndex([])).toMatchObject(new Uint32Array([]));
+  });
+
+  test("Float32Array", () => {
+    expect(makeSortIndex(new Float32Array([3, 2, 1, 0]))).toMatchObject(
+      new Uint32Array([3, 2, 1, 0])
+    );
+    expect(makeSortIndex(new Float32Array([3, 2, 1, 0, 4]))).toMatchObject(
+      new Uint32Array([3, 2, 1, 0, 4])
+    );
+    expect(makeSortIndex(new Float32Array([]))).toMatchObject(
+      new Uint32Array([])
+    );
+  });
+
+  test("Int32Array", () => {
+    expect(makeSortIndex(new Int32Array([3, 2, 1, 0]))).toMatchObject(
+      new Uint32Array([3, 2, 1, 0])
+    );
+    expect(makeSortIndex(new Int32Array([3, 2, 1, 0, 4]))).toMatchObject(
+      new Uint32Array([3, 2, 1, 0, 4])
+    );
+    expect(makeSortIndex(new Int32Array([]))).toMatchObject(
+      new Uint32Array([])
+    );
+  });
+});

client/configuration/babel/babel.dev.js

@@ -0,0 +1,14 @@
+module.exports = {
+  babelrc: false,
+  cacheDirectory: true,
+  presets: [
+    ["modern-browsers", { loose: true, modules: false }],
+    "@babel/preset-react"
+  ],
+  plugins: [
+    "@babel/plugin-proposal-function-bind",
+    "@babel/plugin-proposal-class-properties",
+    ["@babel/plugin-proposal-decorators", { legacy: true }],
+    "@babel/plugin-proposal-export-namespace-from"
+  ]
+};

client/configuration/babel/babel.prod.js

@@ -0,0 +1,15 @@
+module.exports = {
+  babelrc: false,
+  presets: [
+    ["modern-browsers", { loose: true, modules: false }],
+    "@babel/preset-react"
+  ],
+  plugins: [
+    "@babel/plugin-proposal-function-bind",
+    "@babel/plugin-proposal-class-properties",
+    ["@babel/plugin-proposal-decorators", { legacy: true }],
+    "@babel/plugin-proposal-export-namespace-from",
+    "@babel/plugin-transform-react-constant-elements",
+    "@babel/plugin-transform-runtime"
+  ]
+};

client/configuration/eslint/eslint.js

@@ -0,0 +1,58 @@
+module.exports = {
+  root: true,
+  parser: "babel-eslint",
+  extends: ["airbnb", "prettier", "prettier/react"],
+  env: { browser: true, commonjs: true, es6: true },
+  globals: { expect: true },
+  parserOptions: {
+    ecmaVersion: 2017,
+    sourceType: "module",
+    ecmaFeatures: {
+      jsx: true,
+      generators: true
+    }
+  },
+  rules: {
+    "no-magic-numbers": "off",
+    "no-nested-ternary": "off",
+    "func-style": "off",
+    "arrow-parens": "off",
+    "no-use-before-define": "off",
+    "react/jsx-filename-extension": "off",
+    "comma-dangle": "off",
+    "no-underscore-dangle": "off",
+    quotes: ["error", "double"],
+    "implicit-arrow-linebreak": "off",
+    "operator-linebreak": [
+      "error",
+      "after",
+      { overrides: { "?": "before", ":": "before" } }
+    ],
+    "no-console": "off",
+    "spaced-comment": ["error", "always", { exceptions: ["*"] }],
+    "no-param-reassign": "off",
+    "object-curly-newline": ["error", { consistent: true }],
+    "react/prop-types": [0],
+    "space-before-function-paren": "off",
+    "function-paren-newline": "off",
+    "prefer-destructuring": ["error", { object: true, array: false }]
+  },
+  overrides: [
+    {
+      files: ["**/*.test.js"],
+      env: {
+        jest: true // now **/*.test.js files' env has both es6 *and* jest
+      },
+      // Can't extend in overrides: https://github.com/eslint/eslint/issues/8813
+      // "extends": ["plugin:jest/recommended"]
+      plugins: ["jest"],
+      rules: {
+        "jest/no-disabled-tests": "warn",
+        "jest/no-focused-tests": "error",
+        "jest/no-identical-title": "error",
+        "jest/prefer-to-have-length": "warn",
+        "jest/valid-expect": "error"
+      }
+    }
+  ]
+};

client/configuration/webpack/webpack.config.dev.js

@@ -1,41 +1,61 @@
 // jshint esversion: 6
 const path = require("path");
-const autoprefixer = require("autoprefixer");
 const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
 
 const src = path.resolve("src");
 const nodeModules = path.resolve("node_modules");
 
+const babelOptions = require("../babel/babel.dev");
+
 module.exports = {
+  mode: "development",
   devtool: "eval",
-  entry: [
+  entry: ["./src/index"],
-    "webpack-hot-middleware/client?quiet=true&noInfo=true",
-    require.resolve("react-hot-loader/patch"),
-    require.resolve("../polyfills/polyfills"),
-    path.join(src, "index")
-  ],
   output: {
     path: path.resolve("build"),
     pathinfo: true,
     filename: "static/js/bundle.js",
     publicPath: "/"
   },
-  resolve: { extensions: [".js", ".json"] },
   module: {
-    loaders: [
+    rules: [
       {
         test: /\.js$/,
         include: src,
         loader: "babel-loader",
-        query: require("../babel/babel.dev")
+        options: babelOptions
       },
       {
         test: /\.css$/,
-        include: [src, nodeModules],
+        include: src,
-        loader:
+        exclude: [path.resolve(src, "index.css")],
-          "style-loader!css-loader?modules&importLoaders=1&localIdentName=[name]__[local]___[hash:base64:5]!postcss-loader"
+        loader: [
+          {
+            loader: "style-loader"
+          },
+          {
+            loader: "css-loader",
+            options: {
+              modules: true,
+              localIdentName: "[name]__[local]___[hash:base64:5]"
+            }
+          }
+        ]
       },
+      {
+        test: /index\.css$/,
+        include: [path.resolve(src, "index.css")],
+        loader: [
+          {
+            loader: "style-loader"
+          },
+          {
+            loader: "css-loader"
+          }
+        ]
+      },
+
       { test: /\.json$/, include: [src, nodeModules], loader: "json-loader" },
       {
         test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2)(\?.*)?$/,
@@ -57,20 +77,6 @@ module.exports = {
       template: path.resolve("index.html"),
       favicon: path.resolve("favicon.png")
     }),
-    new webpack.LoaderOptionsPlugin({
-      options: {
-        eslint: {
-          configFile: path.resolve("./configuration/eslint/eslint.js"),
-          useEslintrc: false
-        },
-        postcss() {
-          return [autoprefixer];
-        }
-      }
-    }),
-    new webpack.DefinePlugin({ "process.env.NODE_ENV": '"development"' }),
-    // Note: only CSS is currently hot reloaded
-    new webpack.HotModuleReplacementPlugin(),
     new webpack.NoEmitOnErrorsPlugin()
   ]
 };

client/configuration/webpack/webpack.config.prod.js

@@ -1,46 +1,64 @@
 // jshint esversion: 6
 const path = require("path");
-const autoprefixer = require("autoprefixer");
-const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
-const ExtractTextPlugin = require("extract-text-webpack-plugin");
+const MiniCssExtractPlugin = require("mini-css-extract-plugin");
 const SWPrecacheWebpackPlugin = require("sw-precache-webpack-plugin");
-const CopyWebpackPlugin = require("copy-webpack-plugin");
 const HtmlWebpackInlineSourcePlugin = require("html-webpack-inline-source-plugin");
-const MinifyPlugin = require("babel-minify-webpack-plugin");
 
 const src = path.resolve("src");
 const nodeModules = path.resolve("node_modules");
 
-const publicPath = "/";
+const babelOptions = require("../babel/babel.prod");
+
+const publicPath = "";
 
 module.exports = {
+  mode: "production",
   bail: true,
+  cache: false,
   devtool: "cheap-source-map",
-  entry: [require.resolve("../polyfills/polyfills"), path.join(src, "index")],
+  entry: ["./src/index.js"],
   output: {
     path: path.resolve("build"),
     filename: "static/js/[name].[chunkhash:8].js",
-    chunkFilename: "static/js/[name].[chunkhash:8].chunk.js",
     publicPath
   },
-  resolve: { extensions: [".js", ".json"] },
   module: {
-    loaders: [
+    rules: [
       {
         test: /\.js$/,
         include: src,
         loader: "babel-loader",
-        query: require("../babel/babel.prod")
+        options: babelOptions
       },
       {
         test: /\.css$/,
-        include: [src, nodeModules],
+        include: src,
-        loader: ExtractTextPlugin.extract({
+        exclude: [path.resolve(src, "index.css")],
-          fallbackLoader: "style-loader",
+        use: [
-          loader:
+          MiniCssExtractPlugin.loader,
-            "css-loader?modules&importLoaders=1&localIdentName=[name]__[local]___[hash:base64:5]-autoprefixer!postcss-loader"
+          {
-        })
+            loader: "css-loader",
+            options: {
+              modules: true,
+              importLoaders: 1,
+              localIdentName: "[name]__[local]___[hash:base64:5]"
+            }
+          }
+        ]
+      },
+      {
+        test: /index\.css$/,
+        include: [path.resolve(src, "index.css")],
+        use: [
+          MiniCssExtractPlugin.loader,
+          {
+            loader: "css-loader",
+            options: {
+              importLoaders: 1
+            }
+          }
+        ]
       },
       {
         test: /\.json$/,
@@ -52,7 +70,10 @@ module.exports = {
         test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2)(\?.*)?$/,
         include: [src, nodeModules],
         loader: "file-loader",
-        query: { name: "static/media/[name].[hash:8].[ext]" }
+        options: {
+          name: "[name].[hash:8].[ext]",
+          outputPath: "static/media/"
+        }
       },
       {
         test: /\.(mp4|webm)(\?.*)?$/,
@@ -83,24 +104,16 @@ module.exports = {
       }
     }),
     new HtmlWebpackInlineSourcePlugin(),
-    new webpack.LoaderOptionsPlugin({
+    new MiniCssExtractPlugin({
-      options: {
+      filename: "static/css/[name].[contenthash:8].css"
-        eslint: {
-          configFile: path.resolve("./configuration/eslint/eslint.js"),
-          useEslintrc: false
-        },
-        postcss() {
-          return [autoprefixer];
-        }
-      }
     }),
-    new webpack.DefinePlugin({ "process.env.NODE_ENV": '"production"' }),
-    new webpack.optimize.OccurrenceOrderPlugin(),
-    new MinifyPlugin(),
-    new ExtractTextPlugin("static/css/[name].[contenthash:8].css"),
     new SWPrecacheWebpackPlugin({
       cacheId: "cellxgene",
       filename: "service-worker.js"
     })
-  ]
+  ],
+  performance: {
+    maxEntrypointSize: 2000000,
+    maxAssetSize: 2000000
+  }
 };

client/index.html

@@ -3,12 +3,12 @@
   <head>
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1">
-    <title>Cellx</title>
+    <title>cellxgene</title>
-    <link href="https://fonts.googleapis.com/css?family=Roboto:400,400i,700" rel="stylesheet">
+    <link href="https://fonts.googleapis.com/css?family=Roboto+Condensed:400,400i,700" rel="stylesheet">
     <style>
-      html, body, p, h1, h2, h3, h4, h5, h6, span, button, input {
+      html, body, p, h1, h2, h3, h4, h5, h6, span, button, input, label, text, div {
-        font-family: 'Roboto','Helvetica Neue','Helvetica','Arial',sans-serif;
+        font-family: 'Roboto Condensed','Helvetica Neue','Helvetica','Arial',sans-serif;
-        font-size: 16px;
+        font-size: 14px;
       }
       body {
         margin: 0;
@@ -24,7 +24,6 @@
     <noscript>If you're seeing this message, that means <strong>JavaScript has been disabled on your browser</strong>, please <strong>enable JS</strong> to make this app work.</noscript>
 
     <div id="root"></div>
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/d3/4.10.0/d3.js"></script>
+
-    <script src="//d3js.org/d3-scale-chromatic.v0.3.min.js"></script>
   </body>
 </html>

client/index_template.html

@@ -3,11 +3,20 @@
   <head>
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1">
-    <title>Cellx</title>
+    <title>cellxgene</title>
+    <link href="https://fonts.googleapis.com/css?family=Roboto+Condensed:400,400i,700" rel="stylesheet">
     <style>
-      html, body, p, h1, h2, h3, h4, h5, h6, span, button, input {
+      html, body, p, h1, h2, h3, h4, h5, h6, span, button, input, label, text, div {
-        font-family: Helvetica Neue,Helvetica,Arial,sans-serif;
+        font-family: 'Roboto Condensed','Helvetica Neue','Helvetica','Arial',sans-serif;
-        font-size: 16px;
+        font-size: 14px;
+      }
+      body {
+        margin: 0;
+        padding: 0;
+      }
+
+      * {
+        box-sizing: border-box;
       }
     </style>
   </head>
@@ -15,15 +24,12 @@
 <script type="text/javascript">
 window.CELLXGENE = {};
 window.CELLXGENE.API = {
-  prefix: "{{ prefix | safe }}",
+  prefix: window.location.href + "api/",
-  version: "v0.1/"
+  version: "v0.2/"
 };
-window.CELLXGENE.datasetTitle = "{{ datasetTitle }}";
 </script>
     <noscript>If you're seeing this message, that means <strong>JavaScript has been disabled on your browser</strong>, please <strong>enable JS</strong> to make this app work.</noscript>
 
     <div id="root"></div>
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/d3/4.10.0/d3.js"></script>
-    <script src="//d3js.org/d3-scale-chromatic.v0.3.min.js"></script>
   </body>
 </html>

client/package.json

@@ -0,0 +1,146 @@
+{
+  "name": "cellxgene",
+  "version": "0.7.0",
+  "license": "MIT",
+  "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
+  "repository": "https://github.com/chanzuckerberg/cellxgene",
+  "scripts": {
+    "backend-dev": "python3.6 -m venv cellxgene && source cellxgene/bin/activate && yes | pip uninstall cellxgene || true && pip install -e .. && cellxgene launch ",
+    "build": "npm run clean && webpack --config configuration/webpack/webpack.config.prod.js",
+    "clean": "rimraf build",
+    "dev": "npm run clean && webpack --config configuration/webpack/webpack.config.dev.js",
+    "e2e": "jest e2e",
+    "lint": "eslint src",
+    "smoke-test": "start-server-and-test start-server-for-test :5000 e2e",
+    "start": "node server/development.js",
+    "start-server-for-test": "cellxgene launch -p 5000 ../example-dataset/pbmc3k.h5ad",
+    "unit-test": "jest --testPathIgnorePatterns e2e"
+  },
+  "engineStrict": true,
+  "engines": {
+    "npm": ">=3.0.0"
+  },
+  "eslintConfig": {
+    "extends": "./configuration/eslint/eslint.js"
+  },
+  "eslintIgnore": [
+    "src/util/stateManager/matrix_generated.js"
+  ],
+  "resolutions": {
+    "eslint-scope": "3.7.1"
+  },
+  "dependencies": {
+    "@blueprintjs/core": "^3.8.0",
+    "@blueprintjs/icons": "^3.3.0",
+    "@blueprintjs/select": "^3.2.1",
+    "canvas-fit": "^1.5.0",
+    "d3": "^4.10.0",
+    "d3-scale-chromatic": "^1.3.0",
+    "flatbuffers": "^1.10.2",
+    "font-color-contrast": "^1.0.3",
+    "fuzzysort": "^1.1.4",
+    "gl-mat4": "^1.1.4",
+    "gl-matrix": "^2.7.1",
+    "is-number": "^7.0.0",
+    "key-pressed": "0.0.1",
+    "lodash": "^4.17.4",
+    "memoize-one": "^4.0.0",
+    "mouse-position": "^2.0.1",
+    "mouse-pressed": "^1.0.0",
+    "normalize.css": "^8.0.0",
+    "orbit-camera": "^1.0.0",
+    "query-string": "^6.1.0",
+    "react": "^16.6.0",
+    "react-autocomplete": "^1.7.2",
+    "react-dom": "^16.6.0",
+    "react-helmet": "^5.2.0",
+    "react-icons": "^3.2.2",
+    "react-redux": "^5.1.0",
+    "redux": "^4.0.1",
+    "redux-devtools-extension": "^2.13.5",
+    "redux-thunk": "^2.2.0",
+    "regl": "^1.3.9",
+    "scroll-speed": "^1.0.0",
+    "urijs": "^1.19.0"
+  },
+  "devDependencies": {
+    "@babel/core": "^7.1.5",
+    "@babel/plugin-proposal-class-properties": "^7.0.0",
+    "@babel/plugin-proposal-decorators": "^7.0.0",
+    "@babel/plugin-proposal-export-namespace-from": "^7.0.0",
+    "@babel/plugin-proposal-function-bind": "^7.0.0",
+    "@babel/plugin-transform-react-constant-elements": "^7.0.0",
+    "@babel/plugin-transform-runtime": "^7.1.0",
+    "@babel/preset-env": "^7.1.5",
+    "@babel/preset-react": "^7.0.0",
+    "@babel/register": "^7.0.0",
+    "@babel/runtime": "^7.1.5",
+    "babel-core": "^7.0.0-bridge.0",
+    "babel-eslint": "^10.0.1",
+    "babel-jest": "^23.6.0",
+    "babel-loader": "^8.0.0",
+    "babel-preset-modern-browsers": "^12.0.0",
+    "chalk": "^2.4.2",
+    "connect-history-api-fallback": "^1.6.0",
+    "copy-webpack-plugin": "^4.6.0",
+    "css-loader": "^1.0.1",
+    "eslint": "^5.13.0",
+    "eslint-config-airbnb": "^17.1.0",
+    "eslint-config-prettier": "^4.0.0",
+    "eslint-loader": "^2.1.2",
+    "eslint-plugin-filenames": "^1.3.2",
+    "eslint-plugin-import": "^2.16.0",
+    "eslint-plugin-jest": "^22.2.2",
+    "eslint-plugin-jsx-a11y": "^6.2.1",
+    "eslint-plugin-react": "^7.12.4",
+    "express": "^4.14.0",
+    "file-loader": "^2.0.0",
+    "html-webpack-inline-source-plugin": "0.0.10",
+    "html-webpack-plugin": "^3.2.0",
+    "jest": "^24.1.0",
+    "json-loader": "^0.5.4",
+    "mini-css-extract-plugin": "^0.4.1",
+    "puppeteer": "^1.12.1",
+    "rimraf": "^2.6.3",
+    "serve-favicon": "^2.3.0",
+    "start-server-and-test": "^1.7.11",
+    "style-loader": "^0.23.1",
+    "sw-precache-webpack-plugin": "^0.11.5",
+    "url-loader": "^1.1.0",
+    "webpack": "^4.25.1",
+    "webpack-cli": "^3.1.0",
+    "webpack-dev-middleware": "^3.1.3"
+  },
+  "jest": {
+    "testMatch": [
+      "**/__tests__/**/?(*.)(spec|test).js?(x)"
+    ],
+    "testURL": "http://localhost/",
+    "setupFiles": [
+      "./__tests__/setupMissingGlobals.js"
+    ]
+  },
+  "babel": {
+    "env": {
+      "test": {
+        "presets": [
+          "@babel/preset-env",
+          "@babel/preset-react"
+        ],
+        "plugins": [
+          "@babel/plugin-proposal-function-bind",
+          "@babel/plugin-proposal-class-properties",
+          [
+            "@babel/plugin-proposal-decorators",
+            {
+              "legacy": true
+            }
+          ],
+          "@babel/plugin-proposal-export-namespace-from",
+          "@babel/plugin-transform-react-constant-elements",
+          "@babel/plugin-transform-runtime"
+        ]
+      }
+    }
+  }
+}

client/server/development.js

@@ -32,13 +32,11 @@ app.use(historyApiFallback({ verbose: false }));
 
 app.use(
   require("webpack-dev-middleware")(compiler, {
-    noInfo: true,
+    logLevel: "warn",
     publicPath: config.output.publicPath
   })
 );
 
-app.use(require("webpack-hot-middleware")(compiler));
-
 app.use(favicon("./favicon.png"));
 
 app.get("*", (req, res) => {

client/src/actions/index.js

@@ -0,0 +1,333 @@
+// jshint esversion: 6
+import _ from "lodash";
+import * as globals from "../globals";
+import { Universe } from "../util/stateManager";
+import {
+  catchErrorsWrap,
+  doJsonRequest,
+  doBinaryRequest,
+  dispatchNetworkErrorMessageToUser
+} from "../util/actionHelpers";
+
+/*
+Bootstrap application with the initial data loading.
+  * /config - application configuration
+  * /schema - schema of dataframe
+  * /annotations - all metadata annotation
+  * /layout - all default layout
+*/
+const doInitialDataLoad = () =>
+  catchErrorsWrap(async dispatch => {
+    dispatch({ type: "initial data load start" });
+
+    try {
+      const requestJson = _(["config", "schema"])
+        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
+        .map(url => doJsonRequest(url))
+        .value();
+      const requestBinary = _([
+        "annotations/obs",
+        "annotations/var?annotation-name=name",
+        "layout/obs"
+      ])
+        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
+        .map(url => doBinaryRequest(url))
+        .value();
+
+      const results = await Promise.all(_.concat(requestJson, requestBinary));
+
+      /* set config defaults */
+      const config = { ...globals.configDefaults, ...results[0].config };
+      const [, schema, obsAnno, varAnno, obsLayout] = [...results];
+      const universe = Universe.createUniverseFromResponse(
+        config,
+        schema,
+        obsAnno,
+        varAnno,
+        obsLayout
+      );
+
+      dispatch({
+        type: "configuration load complete",
+        config
+      });
+      dispatch({
+        type: "initial data load complete (universe exists)",
+        universe
+      });
+    } catch (error) {
+      dispatch({ type: "initial data load error", error });
+    }
+  }, true);
+
+/*
+Set the view (world) to current selection.   Placeholder for an async action
+which also does re-layout.
+*/
+const regraph = () => (dispatch, getState) => {
+  const { universe, world, crossfilter } = getState().controls;
+  dispatch({
+    type: "set World to current selection",
+    universe,
+    world,
+    crossfilter
+  });
+};
+
+// Throws
+const dispatchExpressionErrors = (dispatch, res) => {
+  const msg = `Unexpected HTTP response while fetching expression data ${
+    res.status
+  }, ${res.statusText}`;
+  dispatchNetworkErrorMessageToUser(msg);
+  throw new Error(msg);
+};
+
+/*
+Fetch expression vectors for each gene in genes.   This is NOT an action
+function, but rather a helper to be called from an action helper that
+needs expression data.
+
+Transparently utilizes cached data if it is already present.
+*/
+async function _doRequestExpressionData(dispatch, getState, genes) {
+  /* helper for this function only */
+  const fetchData = async geneNames => {
+    const res = await fetch(
+      `${globals.API.prefix}${globals.API.version}data/var`,
+      {
+        method: "PUT",
+        body: JSON.stringify({
+          filter: {
+            var: {
+              annotation_value: [{ name: "name", values: geneNames }]
+            }
+          }
+        }),
+        headers: new Headers({
+          accept: "application/octet-stream",
+          "Content-Type": "application/json"
+        })
+      }
+    );
+
+    if (
+      !res.ok ||
+      res.headers.get("Content-Type") !== "application/octet-stream"
+    ) {
+      // WILL throw
+      return dispatchExpressionErrors(dispatch, res);
+    }
+
+    const data = await res.arrayBuffer();
+    return Universe.convertDataFBStoObject(universe, data);
+  };
+
+  const state = getState();
+  const { universe } = state.controls;
+  /* preload data already in cache */
+  let expressionData = _.transform(
+    genes,
+    (expData, g) => {
+      const data = universe.varData.col(g);
+      if (data) {
+        expData[g] = data.asArray();
+      }
+    },
+    {}
+  ); // --> { gene: data }
+  /* make a list of genes for which we do not have data */
+  const genesToFetch = _.filter(genes, g => expressionData[g] === undefined);
+
+  dispatch({ type: "expression load start" });
+
+  /* Fetch data for any genes not in cache */
+  if (genesToFetch.length) {
+    try {
+      const newExpressionData = await fetchData(genesToFetch);
+      expressionData = {
+        ...expressionData,
+        ...newExpressionData
+      };
+    } catch (error) {
+      dispatch({ type: "expression load error", error });
+      throw error; // rethrow
+    }
+  }
+
+  dispatch({ type: "expression load success", expressionData });
+  return expressionData;
+}
+
+function requestSingleGeneExpressionCountsForColoringPOST(gene) {
+  return async (dispatch, getState) => {
+    dispatch({ type: "get single gene expression for coloring started" });
+    try {
+      await _doRequestExpressionData(dispatch, getState, [gene]);
+      const { world } = getState().controls;
+      dispatch({
+        type: "color by expression",
+        gene,
+        data: {
+          [gene]: world.varData.col(gene).asArray()
+        }
+      });
+    } catch (error) {
+      dispatch({
+        type: "get single gene expression for coloring error",
+        error
+      });
+    }
+  };
+}
+
+const requestUserDefinedGene = gene => async (dispatch, getState) => {
+  dispatch({ type: "request user defined gene started" });
+  try {
+    await await _doRequestExpressionData(dispatch, getState, [gene]);
+    const { world } = getState().controls;
+
+    /* then send the success case action through */
+    return dispatch({
+      type: "request user defined gene success",
+      data: {
+        genes: [gene],
+        expression: world.varData.col(gene).asArray()
+      }
+    });
+  } catch (error) {
+    return dispatch({
+      type: "request user defined gene error",
+      error
+    });
+  }
+};
+
+const dispatchDiffExpErrors = (dispatch, response) => {
+  switch (response.status) {
+    case 403:
+      dispatchNetworkErrorMessageToUser(
+        "Too many cells selected for differential experesion calculation - please make a smaller selection."
+      );
+      break;
+    case 501:
+      dispatchNetworkErrorMessageToUser(
+        "Differential expression is not implemented."
+      );
+      break;
+    default: {
+      const msg = `Unexpected differential expression HTTP response ${
+        response.status
+      }, ${response.statusText}`;
+      dispatchNetworkErrorMessageToUser(msg);
+      dispatch({
+        type: "request differential expression error",
+        error: new Error(msg)
+      });
+    }
+  }
+};
+
+const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
+  dispatch,
+  getState
+) => {
+  dispatch({ type: "request differential expression started" });
+  try {
+    /*
+    Steps:
+    1. get the most differentially expressed genes
+    2. get expression data for each
+    */
+    const state = getState();
+    const { universe } = state.controls;
+
+    // Legal values are null, Array or TypedArray.  Null is initial state.
+    if (!set1) set1 = [];
+    if (!set2) set2 = [];
+
+    // These lines ensure that we convert any TypedArray to an Array.
+    // This is necessary because JSON.stringify() does some very strange
+    // things with TypedArrays (they are marshalled to JSON objects, rather
+    // than being marshalled as a JSON array).
+    set1 = Array.isArray(set1) ? set1 : Array.from(set1);
+    set2 = Array.isArray(set2) ? set2 : Array.from(set2);
+
+    const res = await fetch(
+      `${globals.API.prefix}${globals.API.version}diffexp/obs`,
+      {
+        method: "POST",
+        headers: new Headers({
+          Accept: "application/json",
+          "Content-Type": "application/json"
+        }),
+        body: JSON.stringify({
+          mode: "topN",
+          count: num_genes,
+          set1: { filter: { obs: { index: set1 } } },
+          set2: { filter: { obs: { index: set2 } } }
+        })
+      }
+    );
+
+    if (!res.ok || res.headers.get("Content-Type") !== "application/json") {
+      return dispatchDiffExpErrors(dispatch, res);
+    }
+
+    const data = await res.json();
+    // result is [ [varIdx, ...], ... ]
+    const topNGenes = _.map(data, r =>
+      universe.varAnnotations.at(r[0], "name")
+    );
+
+    /*
+    Kick off secondary action to fetch all of the expression data for the
+    topN expressed genes.
+    */
+    await _doRequestExpressionData(dispatch, getState, topNGenes);
+
+    /* then send the success case action through */
+    return dispatch({
+      type: "request differential expression success",
+      data
+    });
+  } catch (error) {
+    return dispatch({
+      type: "request differential expression error",
+      error
+    });
+  }
+};
+
+const resetInterface = () => (dispatch, getState) => {
+  const { universe } = getState().controls;
+
+  dispatch({
+    type: "clear all user defined genes"
+  });
+  dispatch({
+    type: "clear differential expression"
+  });
+  dispatch({
+    type: "reset colorscale"
+  });
+  dispatch({
+    type: "clear scatterplot"
+  });
+  dispatch({
+    type: "reset World to eq Universe",
+    universe
+  });
+  dispatch({
+    type: "increment graph render counter"
+  });
+};
+
+export default {
+  regraph,
+  resetInterface,
+  requestSingleGeneExpressionCountsForColoringPOST,
+  requestDifferentialExpression,
+  requestUserDefinedGene,
+  doInitialDataLoad
+};

client/src/components/app.js

@@ -1,47 +1,37 @@
 // jshint esversion: 6
 import React from "react";
-import _ from "lodash";
 import Helmet from "react-helmet";
-import Container from "./framework/container";
 import { connect } from "react-redux";
-import PulseLoader from "halogen/PulseLoader";
 
+import Container from "./framework/container";
 import LeftSideBar from "./leftsidebar";
-import Parallel from "./continuous/parallel";
+import Legend from "./continuousLegend";
-import Joy from "./joy/joy";
 import Graph from "./graph/graph";
-import * as globals from "../globals";
 import actions from "../actions";
 
-import SectionHeader from "./framework/sectionHeader";
+@connect(state => ({
-
+  loading: state.controls.loading,
-@connect(state => {
+  error: state.controls.error,
-  return {
+  graphRenderCounter: state.controls.graphRenderCounter
-    cells: state.cells
+}))
-  };
-})
 class App extends React.Component {
   constructor(props) {
     super(props);
     this.state = {};
   }
-  _onURLChanged() {
+
-    this.props.dispatch({ type: "url changed", url: document.location.href });
-  }
   componentDidMount() {
+    const { dispatch } = this.props;
+
     /* listen for url changes, fire one when we start the app up */
     window.addEventListener("popstate", this._onURLChanged);
     this._onURLChanged();
 
-    this.props.dispatch(actions.initialize());
+    dispatch(actions.doInitialDataLoad(window.location.search));
 
-    /*
-      first request includes query straight off the url bar for now
-    */
-    this.props.dispatch(actions.requestCells(window.location.search));
     /* listen for resize events */
     window.addEventListener("resize", () => {
-      this.props.dispatch({
+      dispatch({
         type: "window resize",
         data: {
           height: window.innerHeight,
@@ -49,7 +39,7 @@ class App extends React.Component {
         }
       });
     });
-    this.props.dispatch({
+    dispatch({
       type: "window resize",
       data: {
         height: window.innerHeight,
@@ -58,32 +48,31 @@ class App extends React.Component {
     });
   }
 
-  render() {
+  _onURLChanged() {
-    // console.log('app:', this.props, this.state)
+    const { dispatch } = this.props;
 
+    dispatch({ type: "url changed", url: document.location.href });
+  }
+
+  render() {
+    const { loading, error, graphRenderCounter } = this.props;
     return (
       <Container>
         <Helmet title="cellxgene" />
-        {this.props.cells.loading ? (
+        {loading ? (
           <div
-            style={{ position: "fixed", left: window.innerWidth / 2, top: 150 }}
+            style={{
+              position: "fixed",
+              fontWeight: 500,
+              top: window.innerHeight / 2,
+              left: window.innerWidth / 2 - 50
+            }}
           >
-            <PulseLoader color="rgb(0,0,0)" size="10px" margin="4px" />
+            loading cellxgene
-            <span
-              style={{ fontFamily: globals.accentFont, fontStyle: "italic" }}
-            >
-              loading cells
-            </span>
           </div>
         ) : null}
-        {this.props.cells.error ? "Error loading cells" : null}
-        {false ? (
-          <Joy data={this.state.expressions && this.state.expressions.data} />
-        ) : (
-          ""
-        )}
         <div>
-          <LeftSideBar />
+          {loading ? null : <LeftSideBar />}
           <div
             style={{
               padding: 15,
@@ -91,8 +80,8 @@ class App extends React.Component {
               marginLeft: 350 /* but responsive */
             }}
           >
-            <Graph />
+            {loading ? null : <Graph key={graphRenderCounter} />}
-            {/*<Parallel/>*/}
+            <Legend />
           </div>
         </div>
       </Container>

client/src/components/brushableHistogram/index.js

@@ -0,0 +1,375 @@
+/*
+https://bl.ocks.org/mbostock/4341954
+https://bl.ocks.org/mbostock/34f08d5e11952a80609169b7917d4172
+https://bl.ocks.org/SpaceActuary/2f004899ea1b2bd78d6f1dbb2febf771
+*/
+// jshint esversion: 6
+import React from "react";
+import _ from "lodash";
+import { Button, ButtonGroup, Tooltip } from "@blueprintjs/core";
+import { connect } from "react-redux";
+import * as d3 from "d3";
+import memoize from "memoize-one";
+import * as globals from "../../globals";
+import actions from "../../actions";
+import finiteExtent from "../../util/finiteExtent";
+
+@connect(state => ({
+  world: state.controls.world,
+  scatterplotXXaccessor: state.controls.scatterplotXXaccessor,
+  scatterplotYYaccessor: state.controls.scatterplotYYaccessor,
+  crossfilter: state.controls.crossfilter,
+  differential: state.differential,
+  colorAccessor: state.controls.colorAccessor,
+  colorScale: state.controls.colorScale,
+  obsAnnotations: _.get(state.controls.world, "obsAnnotations", null)
+}))
+class HistogramBrush extends React.Component {
+  calcHistogramCache = memoize((obsAnnotations, field, rangeMin, rangeMax) => {
+    const { world } = this.props;
+    const histogramCache = {};
+
+    histogramCache.y = d3
+      .scaleLinear()
+      .range([this.height - this.marginBottom, 0]);
+
+    if (obsAnnotations.hasCol(field)) {
+      // recalculate expensive stuff
+      const allValuesForContinuousFieldAsArray = obsAnnotations
+        .col(field)
+        .asArray();
+
+      histogramCache.x = d3
+        .scaleLinear()
+        .domain([rangeMin, rangeMax])
+        .range([0, this.width]);
+
+      histogramCache.bins = d3
+        .histogram()
+        .domain(histogramCache.x.domain())
+        .thresholds(40)(allValuesForContinuousFieldAsArray);
+
+      histogramCache.numValues = allValuesForContinuousFieldAsArray.length;
+    } else if (world.varData.hasCol(field)) {
+      const varValues = world.varData.col(field).asArray();
+
+      histogramCache.x = d3
+        .scaleLinear()
+        .domain(
+          finiteExtent(varValues)
+        ) /* replace this if we have ranges for genes back from server like we do for annotations on cells */
+        .range([0, this.width]);
+
+      histogramCache.bins = d3
+        .histogram()
+        .domain(histogramCache.x.domain())
+        .thresholds(40)(varValues);
+
+      histogramCache.numValues = varValues.length;
+    }
+
+    return histogramCache;
+  });
+
+  constructor(props) {
+    super(props);
+
+    this.width = 340;
+    this.height = 100;
+    this.marginBottom = 20;
+  }
+
+  componentDidMount() {
+    const { field } = this.props;
+    const { x, y, bins, numValues, svgRef } = this._histogram;
+
+    this.renderAxesBrushBins(x, y, bins, numValues, svgRef, field);
+  }
+
+  componentDidUpdate(prevProps) {
+    const { field, obsAnnotations } = this.props;
+    const { x, y, bins, numValues, svgRef } = this._histogram;
+
+    if (obsAnnotations !== prevProps.obsAnnotations) {
+      this.renderAxesBrushBins(x, y, bins, numValues, svgRef, field);
+    }
+  }
+
+  onBrush(selection, x) {
+    return () => {
+      const { dispatch, field, isObs, isUserDefined, isDiffExp } = this.props;
+
+      if (d3.event.selection) {
+        dispatch({
+          type: "continuous metadata histogram brush",
+          selection: field,
+          continuousNamespace: {
+            isObs,
+            isUserDefined,
+            isDiffExp
+          },
+          range: [x(d3.event.selection[0]), x(d3.event.selection[1])]
+        });
+      } else {
+        dispatch({
+          type: "continuous metadata histogram brush",
+          selection: field,
+          continuousNamespace: {
+            isObs,
+            isUserDefined,
+            isDiffExp
+          },
+          range: null
+        });
+      }
+    };
+  }
+
+  drawHistogram(svgRef) {
+    const { obsAnnotations, field, ranges } = this.props;
+    const histogramCache = this.calcHistogramCache(
+      obsAnnotations,
+      field,
+      ranges.min,
+      ranges.max
+    );
+
+    const { x, y, bins, numValues } = histogramCache;
+
+    this._histogram = { x, y, bins, numValues, svgRef };
+  }
+
+  handleColorAction() {
+    const { obsAnnotations, dispatch, field, world, ranges } = this.props;
+
+    if (obsAnnotations.hasCol(field)) {
+      dispatch({
+        type: "color by continuous metadata",
+        colorAccessor: field,
+        rangeForColorAccessor: ranges
+      });
+    } else if (world.varData.hasCol(field)) {
+      dispatch(actions.requestSingleGeneExpressionCountsForColoringPOST(field));
+    }
+  }
+
+  removeHistogram() {
+    const {
+      dispatch,
+      field,
+      colorAccessor,
+      scatterplotXXaccessor,
+      scatterplotYYaccessor
+    } = this.props;
+    dispatch({
+      type: "clear user defined gene",
+      data: field
+    });
+    if (field === colorAccessor) {
+      dispatch({
+        type: "reset colorscale"
+      });
+    }
+    if (field === scatterplotXXaccessor) {
+      dispatch({
+        type: "set scatterplot x",
+        data: null
+      });
+    }
+    if (field === scatterplotYYaccessor) {
+      dispatch({
+        type: "set scatterplot y",
+        data: null
+      });
+    }
+  }
+
+  handleSetGeneAsScatterplotX() {
+    return () => {
+      const { dispatch, field } = this.props;
+      dispatch({
+        type: "set scatterplot x",
+        data: field
+      });
+    };
+  }
+
+  handleSetGeneAsScatterplotY() {
+    return () => {
+      const { dispatch, field } = this.props;
+      dispatch({
+        type: "set scatterplot y",
+        data: field
+      });
+    };
+  }
+
+  renderAxesBrushBins(x, y, bins, numValues, svgRef, field) {
+    /* Remove everything */
+    d3.select(svgRef)
+      .selectAll("*")
+      .remove();
+
+    /* BINS */
+    d3.select(svgRef)
+      .insert("g", "*")
+      .attr("fill", "#bbb")
+      .selectAll("rect")
+      .data(bins)
+      .enter()
+      .append("rect")
+      .attr("class", "bar")
+      .attr("x", d => x(d.x0) + 1)
+      .attr("y", d => y(d.length / numValues))
+      .attr("width", d => Math.abs(x(d.x1) - x(d.x0) - 1))
+      .attr("height", d => y(0) - y(d.length / numValues));
+
+    /* BRUSH */
+    d3.select(svgRef)
+      .append("g")
+      .attr("class", "brush")
+      .attr("data-testid", `${svgRef.id}-brush`)
+      .call(
+        d3
+          .brushX()
+          .on("brush", this.onBrush(field, x.invert).bind(this))
+          .on("end", this.onBrush(field, x.invert).bind(this))
+      );
+
+    /* AXIS */
+    d3.select(svgRef)
+      .append("g")
+      .attr("class", "axis axis--x")
+      .attr("transform", `translate(0,${this.height - this.marginBottom})`)
+      .call(d3.axisBottom(x).ticks(5));
+
+    d3.select(svgRef)
+      .selectAll(".axis--x text")
+      .style("fill", "rgb(80,80,80)");
+
+    d3.select(svgRef)
+      .selectAll(".axis--x path")
+      .style("stroke", "rgb(230,230,230)");
+
+    d3.select(svgRef)
+      .selectAll(".axis--x line")
+      .style("stroke", "rgb(230,230,230)");
+  }
+
+  render() {
+    const {
+      field,
+      colorAccessor,
+      isUserDefined,
+      isDiffExp,
+      logFoldChange,
+      pval,
+      pvalAdj,
+      scatterplotXXaccessor,
+      scatterplotYYaccessor,
+      zebra
+    } = this.props;
+
+    return (
+      <div
+        id={`histogram_${field}`}
+        data-testid={`histogram-${field}`}
+        data-testclass={isDiffExp ? `histogram-diffexp` : ""}
+        style={{
+          padding: globals.leftSidebarSectionPadding,
+          backgroundColor: zebra ? globals.lightestGrey : "white"
+        }}
+      >
+        <div style={{ display: "flex", justifyContent: "flex-end" }}>
+          {isDiffExp || isUserDefined ? (
+            <span>
+              <span
+                style={{ marginRight: 7 }}
+                className="bp3-icon-standard bp3-icon-scatter-plot"
+              />
+              <ButtonGroup style={{ marginRight: 7 }}>
+                <Button
+                  onClick={this.handleSetGeneAsScatterplotX(field).bind(this)}
+                  active={scatterplotXXaccessor === field}
+                  intent={scatterplotXXaccessor === field ? "primary" : "none"}
+                >
+                  plot x
+                </Button>
+                <Button
+                  onClick={this.handleSetGeneAsScatterplotY(field).bind(this)}
+                  active={scatterplotYYaccessor === field}
+                  intent={scatterplotYYaccessor === field ? "primary" : "none"}
+                >
+                  plot y
+                </Button>
+              </ButtonGroup>
+            </span>
+          ) : null}
+          {isUserDefined ? (
+            <Button
+              minimal
+              onClick={this.removeHistogram.bind(this)}
+              style={{
+                color: globals.blue,
+                cursor: "pointer",
+                marginLeft: 7
+              }}
+            >
+              remove
+            </Button>
+          ) : null}
+          <Tooltip content="Use as color scale" position="bottom">
+            <Button
+              onClick={this.handleColorAction.bind(this)}
+              active={colorAccessor === field}
+              intent={colorAccessor === field ? "primary" : "none"}
+              icon="tint"
+            />
+          </Tooltip>
+        </div>
+        <svg
+          width={this.width}
+          height={this.height}
+          id={`histogram_${field}_svg`}
+          ref={svgRef => {
+            this.drawHistogram(svgRef);
+          }}
+        />
+        <div
+          style={{
+            display: "flex",
+            justifyContent: "center"
+          }}
+        >
+          <span style={{ fontStyle: "italic" }}>{field}</span>
+        </div>
+
+        {isDiffExp ? (
+          <div
+            style={{
+              display: "flex",
+              justifyContent: "center",
+              alignItems: "baseline"
+            }}
+          >
+            <span>
+              <strong>log fold change:</strong>
+              {` ${logFoldChange.toPrecision(4)}`}
+            </span>
+            <span
+              style={{
+                marginLeft: 7,
+                padding: 2
+              }}
+            >
+              <strong>p-value (adj):</strong>
+              {pvalAdj < 0.0001 ? " < 0.0001" : ` ${pvalAdj.toFixed(4)}`}
+            </span>
+          </div>
+        ) : null}
+      </div>
+    );
+  }
+}
+
+export default HistogramBrush;

client/src/components/categorical/categorical.js

@@ -0,0 +1,37 @@
+// jshint esversion: 6
+import React from "react";
+import _ from "lodash";
+import { connect } from "react-redux";
+import * as globals from "../../globals";
+import Category from "./category";
+
+@connect(state => ({
+  categoricalSelectionState: state.controls.categoricalSelectionState
+}))
+class Categories extends React.Component {
+  render() {
+    const { categoricalSelectionState } = this.props;
+    if (!categoricalSelectionState) return null;
+
+    return (
+      <div
+        style={{
+          padding: globals.leftSidebarSectionPadding
+        }}
+      >
+        <p
+          style={Object.assign({}, globals.leftSidebarSectionHeading, {
+            marginTop: 4
+          })}
+        >
+          Categorical Metadata
+        </p>
+        {_.map(categoricalSelectionState, (catState, catName) => (
+          <Category key={catName} metadataField={catName} />
+        ))}
+      </div>
+    );
+  }
+}
+
+export default Categories;

client/src/components/categorical/category.js

@@ -0,0 +1,182 @@
+import React from "react";
+import _ from "lodash";
+import { connect } from "react-redux";
+import { FaChevronRight, FaChevronDown } from "react-icons/fa";
+import { Button, Tooltip } from "@blueprintjs/core";
+
+import * as globals from "../../globals";
+import Value from "./value";
+import sortedCategoryValues from "./util";
+
+@connect(state => ({
+  colorAccessor: state.controls.colorAccessor,
+  categoricalSelectionState: state.controls.categoricalSelectionState
+}))
+class Category extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = {
+      isChecked: true,
+      isExpanded: false
+    };
+  }
+
+  componentDidUpdate() {
+    const { categoricalSelectionState, metadataField } = this.props;
+    const cat = categoricalSelectionState[metadataField];
+    const categoryCount = {
+      // total number of categories in this dimension
+      totalCatCount: cat.numCategories,
+      // number of selected options in this category
+      selectedCatCount: _.reduce(
+        cat.categorySelected,
+        (res, cond) => (cond ? res + 1 : res),
+        0
+      )
+    };
+    if (categoryCount.selectedCatCount === categoryCount.totalCatCount) {
+      /* everything is on, so not indeterminate */
+      this.checkbox.indeterminate = false;
+    } else if (categoryCount.selectedCatCount === 0) {
+      /* nothing is on, so no */
+      this.checkbox.indeterminate = false;
+    } else if (categoryCount.selectedCatCount < categoryCount.totalCatCount) {
+      /* to be explicit... */
+      this.checkbox.indeterminate = true;
+    }
+  }
+
+  handleColorChange() {
+    const { dispatch, metadataField } = this.props;
+    dispatch({
+      type: "color by categorical metadata",
+      colorAccessor: metadataField
+    });
+  }
+
+  toggleAll() {
+    const { dispatch, metadataField } = this.props;
+    dispatch({
+      type: "categorical metadata filter all of these",
+      metadataField
+    });
+    this.setState({ isChecked: true });
+  }
+
+  toggleNone() {
+    const { dispatch, metadataField } = this.props;
+    dispatch({
+      type: "categorical metadata filter none of these",
+      metadataField
+    });
+    this.setState({ isChecked: false });
+  }
+
+  handleToggleAllClick() {
+    const { isChecked } = this.state;
+    // || this.checkbox.indeterminate === false
+    if (isChecked) {
+      this.toggleNone();
+    } else if (!isChecked) {
+      this.toggleAll();
+    }
+  }
+
+  renderCategoryItems() {
+    const { categoricalSelectionState, metadataField } = this.props;
+
+    const cat = categoricalSelectionState[metadataField];
+    const optTuples = sortedCategoryValues([...cat.categoryIndices]);
+    return _.map(optTuples, (tuple, i) => (
+      <Value
+        optTuples={optTuples}
+        key={tuple[1]}
+        metadataField={metadataField}
+        categoryIndex={tuple[1]}
+        i={i}
+      />
+    ));
+  }
+
+  render() {
+    const { isExpanded, isChecked } = this.state;
+    const {
+      metadataField,
+      colorAccessor,
+      categoricalSelectionState
+    } = this.props;
+    const { isTruncated } = categoricalSelectionState[metadataField];
+    return (
+      <div
+        style={{
+          maxWidth: globals.maxControlsWidth
+        }}
+      >
+        <div
+          style={{
+            display: "flex",
+            justifyContent: "space-between",
+            alignItems: "baseline"
+          }}
+        >
+          <div
+            style={{
+              display: "flex",
+              justifyContent: "flex-start",
+              alignItems: "baseline"
+            }}
+          >
+            <label className="bp3-control bp3-checkbox">
+              <input
+                onChange={this.handleToggleAllClick.bind(this)}
+                ref={el => {
+                  this.checkbox = el;
+                  return el;
+                }}
+                checked={isChecked}
+                type="checkbox"
+              />
+              <span className="bp3-control-indicator" />
+              {""}
+            </label>
+
+            <span
+              style={{
+                cursor: "pointer",
+                display: "inline-block"
+              }}
+              onClick={() => {
+                this.setState({ isExpanded: !isExpanded });
+              }}
+            >
+              {metadataField}
+              {isExpanded ? (
+                <FaChevronDown style={{ fontSize: 10, marginLeft: 5 }} />
+              ) : (
+                <FaChevronRight style={{ fontSize: 10, marginLeft: 5 }} />
+              )}
+            </span>
+          </div>
+          <Tooltip content="Use as color scale" position="bottom">
+            <Button
+              onClick={this.handleColorChange.bind(this)}
+              active={colorAccessor === metadataField}
+              intent={colorAccessor === metadataField ? "primary" : "none"}
+              icon={"tint"}
+            />
+          </Tooltip>
+        </div>
+        <div style={{ marginLeft: 26 }}>
+          {isExpanded ? this.renderCategoryItems() : null}
+        </div>
+        <div>
+          {isExpanded && isTruncated ? (
+            <p style={{ paddingLeft: 15 }}>... truncated list ...</p>
+          ) : null}
+        </div>
+      </div>
+    );
+  }
+}
+
+export default Category;

client/src/components/categorical/occupancy.js

@@ -0,0 +1,73 @@
+// jshint esversion: 6
+import React from "react";
+import _ from "lodash";
+import { connect } from "react-redux";
+import * as d3 from "d3";
+
+@connect()
+class Occupancy extends React.Component {
+  render() {
+    const {
+      occupancy,
+      colorScale,
+      categoricalSelectionState,
+      colorAccessor,
+      schema
+    } = this.props;
+    const width = 100;
+    const height = 11;
+
+    const categories = _.filter(schema.annotations.obs, {
+      name: colorAccessor
+    })[0].categories;
+
+    const x = d3
+      .scaleLinear()
+      /* get all the keys d[1] as an array, then find the sum */
+      .domain([0, d3.sum(Array.from(occupancy, d => d[1]))])
+      .range([0, width]);
+
+    let currentOffset = 0;
+
+    const stacks = categoricalSelectionState[colorAccessor].categoryValues.map(
+      d => {
+        const o = occupancy.get(d);
+
+        const scaledValue = x(o);
+
+        const stackItem = {
+          key: d,
+          value: o || 0,
+          rectWidth: o ? scaledValue : 0,
+          offset: currentOffset,
+          fill: o ? colorScale(categories.indexOf(d)) : "rgb(255,255,255)"
+        };
+        currentOffset += o ? scaledValue : 0;
+        return stackItem;
+      }
+    );
+
+    return (
+      <svg
+        style={{
+          marginRight: 5,
+          width,
+          height
+        }}
+      >
+        {stacks.map(d => (
+          <rect
+            key={d.key}
+            width={d.rectWidth}
+            height={height}
+            x={d.offset}
+            title={d.metadataField}
+            fill={d.fill}
+          />
+        ))}
+      </svg>
+    );
+  }
+}
+
+export default Occupancy;

client/src/components/categorical/util.js

@@ -0,0 +1,35 @@
+// jshint esversion: 6
+
+// values is [ [optVal, optIdx], ...]
+// index is range array
+// return sorted index
+
+import isNumber from "is-number";
+import _ from "lodash";
+
+const sortedCategoryValues = values => {
+  /* this sort could be memoized for perf */
+
+  const strings = [];
+  const ints = [];
+
+  _.forEach(values, v => {
+    if (isNumber(v[0])) {
+      ints.push(v);
+    } else {
+      strings.push(v);
+    }
+  });
+
+  strings.sort((a, b) => {
+    const textA = String(a[0]).toUpperCase();
+    const textB = String(b[0]).toUpperCase();
+    return textA < textB ? -1 : textA > textB ? 1 : 0;
+  });
+
+  ints.sort((a, b) => +a[0] - +b[0]);
+
+  return ints.concat(strings);
+};
+
+export default sortedCategoryValues;

client/src/components/categorical/value.js

@@ -0,0 +1,142 @@
+// jshint esversion: 6
+import { connect } from "react-redux";
+import React from "react";
+import _ from "lodash";
+import Occupancy from "./occupancy";
+import { countCategoryValues2D } from "../../util/stateManager/worldUtil";
+import * as globals from "../../globals";
+
+@connect(state => ({
+  categoricalSelectionState: state.controls.categoricalSelectionState,
+  colorScale: state.controls.colorScale,
+  colorAccessor: state.controls.colorAccessor,
+  schema: _.get(state.controls.world, "schema", null),
+  world: state.controls.world
+}))
+class CategoryValue extends React.Component {
+  toggleOff() {
+    const { dispatch, metadataField, categoryIndex } = this.props;
+    dispatch({
+      type: "categorical metadata filter deselect",
+      metadataField,
+      categoryIndex
+    });
+  }
+
+  toggleOn() {
+    const { dispatch, metadataField, categoryIndex } = this.props;
+    dispatch({
+      type: "categorical metadata filter select",
+      metadataField,
+      categoryIndex
+    });
+  }
+
+  render() {
+    const {
+      categoricalSelectionState,
+      metadataField,
+      categoryIndex,
+      colorAccessor,
+      colorScale,
+      i,
+      schema,
+      world
+    } = this.props;
+
+    if (!categoricalSelectionState) return null;
+
+    const category = categoricalSelectionState[metadataField];
+    const selected = category.categorySelected[categoryIndex];
+    const count = category.categoryCounts[categoryIndex];
+    const value = category.categoryValues[categoryIndex];
+    const displayString = String(
+      category.categoryValues[categoryIndex]
+    ).valueOf();
+
+    /* this is the color scale, so add swatches below */
+    const isColorBy = metadataField === colorAccessor;
+    let categories = null;
+    let occupancy = null;
+
+    if (isColorBy && schema) {
+      categories = _.filter(schema.annotations.obs, {
+        name: colorAccessor
+      })[0].categories;
+    }
+
+    if (
+      colorAccessor &&
+      !isColorBy &&
+      categoricalSelectionState[colorAccessor]
+    ) {
+      occupancy = countCategoryValues2D(
+        metadataField,
+        colorAccessor,
+        world.obsAnnotations
+      );
+    }
+
+    return (
+      <div
+        key={i}
+        style={{
+          display: "flex",
+          alignItems: "baseline",
+          justifyContent: "space-between"
+        }}
+      >
+        <div
+          style={{
+            margin: 0,
+            padding: 0,
+            userSelect: "none",
+            width: globals.leftSidebarWidth - 130,
+            display: "flex",
+            justifyContent: "space-between"
+          }}
+        >
+          <label className="bp3-control bp3-checkbox">
+            <input
+              onChange={
+                selected ? this.toggleOff.bind(this) : this.toggleOn.bind(this)
+              }
+              checked={selected}
+              type="checkbox"
+            />
+            <span className="bp3-control-indicator" />
+            {displayString}
+          </label>
+          <span style={{ flexShrink: 0 }}>
+            {colorAccessor &&
+            !isColorBy &&
+            categoricalSelectionState[colorAccessor] ? (
+              <Occupancy
+                occupancy={occupancy.get(
+                  category.categoryValues[categoryIndex]
+                )}
+                {...this.props}
+              />
+            ) : null}
+          </span>
+        </div>
+        <span>
+          <span>{count}</span>
+          <svg
+            style={{
+              marginLeft: 5,
+              width: 11,
+              height: 11,
+              backgroundColor:
+                isColorBy && categories
+                  ? colorScale(categories.indexOf(value))
+                  : "inherit"
+            }}
+          />
+        </span>
+      </div>
+    );
+  }
+}
+
+export default CategoryValue;

client/src/components/continuous/continuous.js

@@ -0,0 +1,99 @@
+// jshint esversion: 6
+/* rc slider https://www.npmjs.com/package/rc-slider */
+
+import React from "react";
+import _ from "lodash";
+import { connect } from "react-redux";
+import * as globals from "../../globals";
+
+import HistogramBrush from "../brushableHistogram";
+
+@connect(state => ({
+  obsAnnotations: _.get(state.controls.world, "obsAnnotations", null),
+  colorAccessor: state.controls.colorAccessor,
+  colorScale: state.controls.colorScale,
+  selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null),
+  schema: _.get(state.controls.world, "schema", null)
+}))
+class Continuous extends React.Component {
+  constructor(props) {
+    super(props);
+    this.hasContinuous = false;
+    this.continuousChecked = false;
+
+    this.state = {};
+  }
+
+  componentDidUpdate() {}
+
+  handleColorAction(key) {
+    return () => {
+      const { dispatch, obsAnnotations } = this.props;
+      const summary = obsAnnotations.col(key).summarize();
+      dispatch({
+        type: "color by continuous metadata",
+        colorAccessor: key,
+        rangeForColorAccessor: summary
+      });
+    };
+  }
+
+  render() {
+    const { obsAnnotations, schema } = this.props;
+    if (schema && !this.continuousChecked) {
+      this.hasContinuous = _.some(
+        schema.annotations.obs,
+        d => d.type === "int32" || d.type === "float32"
+      );
+      this.continuousChecked = true; /* only do this once */
+    }
+
+    /* initial value for iterator to simulate index, ranges is an object */
+    let zebra = -1;
+
+    return (
+      <div>
+        {this.hasContinuous ? (
+          <p
+            style={Object.assign({}, globals.leftSidebarSectionHeading, {
+              marginTop: 40,
+              paddingLeft: globals.leftSidebarSectionPadding
+            })}
+          >
+            Continuous metadata
+          </p>
+        ) : null}
+        {obsAnnotations
+          ? _.map(obsAnnotations.colIndex.keys(), key => {
+              const summary = obsAnnotations.col(key).summarize();
+              const isColorField =
+                key.includes("color") || key.includes("Color");
+              const nonFiniteExtent =
+                summary.min === undefined || summary.max === undefined;
+              zebra += 1;
+              if (
+                !summary.categorical &&
+                key !== "name" &&
+                !isColorField &&
+                !nonFiniteExtent
+              ) {
+                return (
+                  <HistogramBrush
+                    key={key}
+                    field={key}
+                    isObs
+                    zebra={zebra % 2 === 0}
+                    ranges={summary}
+                    handleColorAction={this.handleColorAction(key).bind(this)}
+                  />
+                );
+              }
+              return null;
+            })
+          : null}
+      </div>
+    );
+  }
+}
+
+export default Continuous;

client/src/components/continuous/drawAxes.js

@@ -1,4 +1,5 @@
 // jshint esversion: 6
+import * as d3 from "d3";
 import styles from "./parallelCoordinates.css";
 import { yAxis, brushstart } from "./util";
 
@@ -19,13 +20,11 @@ const drawAxes = (
   ******************************************/
 
   function brush() {
-    var actives = [];
+    const actives = [];
     svg
       .selectAll(".parcoords_axis .parcoords_brush")
-      .filter(function(d) {
+      .filter(() => d3.brushSelection(this))
-        return d3.brushSelection(this);
+      .each(d => {
-      })
-      .each(function(d) {
         actives.push({
           dimension: d,
           extent: d3.brushSelection(this)
@@ -35,20 +34,18 @@ const drawAxes = (
     handleBrushAction(actives);
   }
 
-  var axes = svg
+  const axes = svg
     .selectAll(".parcoords_axis")
     .data(dimensions)
     .enter()
     .append("g")
     .attr("class", `${styles.axis} parcoords_axis`)
-    .attr("transform", (d, i) => {
+    .attr("transform", (d, i) => `translate(${xscale(i)})`);
-      return "translate(" + xscale(i) + ")";
-    });
 
   axes
     .append("g")
-    .each(function(d) {
+    .each(d => {
-      var renderAxis =
+      const renderAxis =
         "axis" in d
           ? d.axis.scale(d.scale) // custom axis
           : yAxis.scale(d.scale); // default axis
@@ -60,15 +57,13 @@ const drawAxes = (
     })
     .attr("class", styles.title)
     .attr("text-anchor", "start")
-    .text(function(d) {
+    .text(d => ("description" in d ? `${d.description}  🖌️` : `${d.key}  🖌️`));
-      return "description" in d ? d.description + "  🖌️" : d.key + "  🖌️";
-    });
 
   // Add and store a brush for each axis.
   axes
     .append("g")
     .attr("class", `${styles.brush} parcoords_brush`)
-    .each(function(d) {
+    .each(d => {
       d3.select(this).call(
         (d.brush = d3
           .brushY()

client/src/components/continuous/parallelCoordinates.css

@@ -7,24 +7,24 @@ body {
 }
 */
 
-.parcoords {
+:local(.parcoords) {
   display: block;
 }
 
-.parcoords svg,
+:local(.parcoords) svg,
-.parcoords canvas {
+:local(.parcoords) canvas {
   font: 10px sans-serif;
   position: absolute;
 }
 
-.parcoords canvas {
+:local(.parcoords) canvas {
   opacity: 0.9;
   pointer-events: none;
 }
 
-.axis .title {
+:local(.axis .title) {
   font-size: 10px;
-  transform: rotate(-21deg) translate(-5px,-6px);
+  transform: rotate(-21deg) translate(-5px, -6px);
   fill: #222;
   cursor: pointer;
 }
@@ -32,14 +32,14 @@ body {
 /* -webkit-filter: grayscale(100%);
  filter: grayscale(100%); */
 
-.axis line,
+:local(.axis) line,
-.axis path {
+:local(.axis) path {
   fill: none;
   stroke: #ccc;
   stroke-width: 1px;
 }
 
-.axis .tick text {
+:local(.axis .tick) text {
   fill: #222;
   pointer-events: none;
 }
@@ -52,39 +52,39 @@ old, from reference bl.ocks
 }
 */
 
-.axis:hover line,
+:local(.axis:hover) line,
-.axis:hover path,
+:local(.axis:hover) path,
-.axis.active line,
+:local(.axis.active) line,
-.axis.active path {
+:local(.axis.active) path {
   fill: none;
   stroke: #222;
   stroke-width: 1px;
 }
 
-.axis:hover .title {
+:local(.axis:hover .title) {
   font-weight: bold;
 }
 
-.axis:hover .tick text {
+:local(.axis:hover .tick) text {
   opacity: 1;
 }
 
-.axis.active .title {
+:local(.axis.active .title) {
   font-weight: bold;
 }
 
-.axis.active .tick text {
+:local(.axis.active .tick) text {
   opacity: 1;
   font-weight: bold;
 }
 
-.brush .extent {
+:local(.brush .extent) {
-  fill-opacity: .3;
+  fill-opacity: 0.3;
   stroke: #fff;
   stroke-width: 1px;
 }
 
-.pre {
+:local(.pre) {
   width: 100%;
   height: 300px;
   margin: 6px 12px;

client/src/components/continuous/setupParallelCoordinates.js

@@ -4,26 +4,28 @@
 ******************************************
 ******************************************/
 // jshint esversion: 6
-const setupParallelCoordinates = (width, height, margin) => {
+import * as d3 from "d3";
-  var container = d3.select("#parcoords");
 
-  var svg = container
+const setupParallelCoordinates = (width, height, margin) => {
+  const container = d3.select("#parcoords");
+
+  const svg = container
     .append("svg")
     .attr("width", width + margin.left + margin.right)
     .attr("height", height + margin.top + margin.bottom)
     .append("g")
-    .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
+    .attr("transform", `translate(${margin.left},${margin.top})`);
 
-  var canvas = container
+  const canvas = container
     .append("canvas")
     .attr("width", width * devicePixelRatio)
     .attr("height", height * devicePixelRatio)
-    .style("width", width + "px")
+    .style("width", `${width}px`)
-    .style("height", height + "px")
+    .style("height", `${height}px`)
-    .style("margin-top", margin.top + "px")
+    .style("margin-top", `${margin.top}px`)
-    .style("margin-left", margin.left + "px");
+    .style("margin-left", `${margin.left}px`);
 
-  var ctx = canvas.node().getContext("2d");
+  const ctx = canvas.node().getContext("2d");
   ctx.globalCompositeOperation = "darken";
   ctx.globalAlpha = 0.15;
   ctx.lineWidth = 1.5;

client/src/components/continuous/util.js

@@ -1,5 +1,6 @@
 // jshint esversion: 6
 import _ from "lodash";
+import * as d3 from "d3";
 
 const paddingRight = 120;
 const continuousChartWidth = 1200;
@@ -17,11 +18,10 @@ export const createDimensions = data => {
   _.each(data, (value, key) => {
     if (value.range) {
       newArr.push({
-        key: key /* room for confusion: lodash calls this key, it's also the name of the property parallel coords code is looking for */,
+        key /* room for confusion: lodash calls this key, it's also the name of the property parallel coords code is looking for */,
         type: {
-          within: (d, extent, dim) => {
+          within: (d, extent, dim) =>
-            return extent[0] <= dim.scale(d) && dim.scale(d) <= extent[1];
+            extent[0] <= dim.scale(d) && dim.scale(d) <= extent[1]
-          }
         },
         scale: d3
           .scaleSqrt()
@@ -39,19 +39,12 @@ export const brushstart = () => {
   d3.event.sourceEvent.stopPropagation();
 };
 
-export const d3_functor = v => {
+// Unused.
-  return typeof v === "function"
+// export const d3_functor = v => (typeof v === "function" ? v : () => v);
-    ? v
+export const project = (d, dimensions, xscale) =>
-    : () => {
+  dimensions.map((p, i) => {
-        return v;
-      };
-};
-
-export const project = (d, dimensions, xscale) => {
-  return dimensions.map((p, i) => {
     // check if data element has property and contains a value
     if (!(p.key in d) || d[p.key] === null) return null;
 
     return [xscale(i), p.scale(d[p.key])];
   });
-};

client/src/components/continuousLegend/index.js

@@ -0,0 +1,152 @@
+// jshint esversion: 6
+import React from "react";
+import { connect } from "react-redux";
+import * as d3 from "d3";
+import { interpolateViridis, interpolateCool } from "d3-scale-chromatic";
+
+// create continuous color legend
+// http://bl.ocks.org/syntagmatic/e8ccca52559796be775553b467593a9f
+const continuous = (selectorId, colorscale, colorAccessor) => {
+  const legendheight = 200;
+  const legendwidth = 80;
+  const margin = { top: 10, right: 60, bottom: 10, left: 2 };
+
+  const canvas = d3
+    .select(selectorId)
+    .style("height", `${legendheight}px`)
+    .style("width", `${legendwidth}px`)
+    .append("canvas")
+    .attr("height", legendheight - margin.top - margin.bottom)
+    .attr("width", 1)
+    .style("height", `${legendheight - margin.top - margin.bottom}px`)
+    .style("width", `${legendwidth - margin.left - margin.right}px`)
+    .style("position", "absolute")
+    .style("top", `${margin.top + 1}px`)
+    .style("left", `${margin.left + 1}px`)
+    .style(
+      "transform",
+      "scale(1,-1)"
+    ) /* flip it! dark is high value light is low.
+    we flip the color scale as well [1, 0] instead of [0, 1] */
+    .node();
+
+  const ctx = canvas.getContext("2d");
+
+  const legendscale = d3
+    .scaleLinear()
+    .range([1, legendheight - margin.top - margin.bottom])
+    .domain([
+      colorscale.domain()[1],
+      colorscale.domain()[0]
+    ]); /* we flip this to make viridis colors dark if high in the color scale */
+
+  // image data hackery based on http://bl.ocks.org/mbostock/048d21cf747371b11884f75ad896e5a5
+  const image = ctx.createImageData(1, legendheight);
+  d3.range(legendheight).forEach(i => {
+    const c = d3.rgb(colorscale(legendscale.invert(i)));
+    image.data[4 * i] = c.r;
+    image.data[4 * i + 1] = c.g;
+    image.data[4 * i + 2] = c.b;
+    image.data[4 * i + 3] = 255;
+  });
+  ctx.putImageData(image, 0, 0);
+
+  // A simpler way to do the above, but possibly slower. keep in mind the legend
+  // width is stretched because the width attr of the canvas is 1
+  // See http://stackoverflow.com/questions/4899799/whats-the-best-way-to-set-a-single-pixel-in-an-html5-canvas
+  /*
+  d3.range(legendheight).forEach(function(i) {
+    ctx.fillStyle = colorscale(legendscale.invert(i));
+    ctx.fillRect(0,i,1,1);
+  });
+  */
+
+  const legendaxis = d3
+    .axisRight()
+    .scale(legendscale)
+    .tickSize(6)
+    .ticks(8);
+
+  const svg = d3
+    .select(selectorId)
+    .append("svg")
+    .attr("height", `${legendheight}px`)
+    .attr("width", `${legendwidth}px`)
+    .style("position", "absolute")
+    .style("left", "0px")
+    .style("top", "0px");
+
+  svg
+    .append("g")
+    .attr("class", "axis")
+    .attr(
+      "transform",
+      `translate(${legendwidth - margin.left - margin.right + 3},${margin.top})`
+    )
+    .call(legendaxis);
+
+  // text label for the y axis
+  svg
+    .append("text")
+    .attr("transform", "rotate(-90)")
+    .attr("y", 2)
+    .attr("x", 0 - legendheight / 2)
+    .attr("dy", "1em")
+    .style("text-anchor", "middle")
+    .style("fill", "white")
+    .text(colorAccessor);
+};
+
+@connect(state => ({
+  colorAccessor: state.controls.colorAccessor,
+  colorScale: state.controls.colorScale,
+  responsive: state.responsive
+}))
+class ContinuousLegend extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = {};
+  }
+
+  componentDidUpdate(prevProps) {
+    const { colorAccessor, responsive, colorScale } = this.props;
+    if (
+      prevProps.colorAccessor !== colorAccessor ||
+      prevProps.responsive.height !== responsive.height ||
+      prevProps.responsive.width !== responsive.width
+    ) {
+      /* always remove it, if it's not continuous we don't put it back. */
+      d3.select("#continuous_legend")
+        .selectAll("*")
+        .remove();
+    }
+
+    if (colorAccessor && colorScale && colorScale.range) {
+      /* fragile! continuous range is 0 to 1, not [#fa4b2c, ...], make this a flag? */
+      if (colorScale.range()[0][0] !== "#") {
+        continuous(
+          "#continuous_legend",
+          d3.scaleSequential(interpolateCool).domain(colorScale.domain()),
+          colorAccessor
+        );
+      }
+    }
+  }
+
+  render() {
+    const { colorAccessor, responsive } = this.props;
+    return (
+      <div
+        id="continuous_legend"
+        style={{
+          position: "fixed",
+          display: colorAccessor ? "inherit" : "none",
+          right: 0,
+          top: responsive.height / 2
+        }}
+      />
+    );
+  }
+}
+
+export default ContinuousLegend;

client/src/components/framework/buttons.css

@@ -1,4 +1,4 @@
-.btn {
+:local(.btn) {
   background: none;
   cursor: pointer;
   color: "#FFF";

client/src/components/framework/container.css

@@ -1,3 +1,3 @@
-.container {
+:local(.container) {
   min-height: 100%;
 }

client/src/components/framework/container.js

@@ -0,0 +1,11 @@
+// jshint esversion: 6
+import React from "react";
+
+import styles from "./container.css";
+
+const Container = props => {
+  const { children } = props;
+  return <div className={styles.container}>{children}</div>;
+};
+
+export default Container;

client/src/components/framework/toasters.js

@@ -0,0 +1,31 @@
+import { Position, Toaster, Intent } from "@blueprintjs/core";
+
+/** Singleton toaster instance. Create separate instances for different options. */
+
+const ErrorToastTopCenter = Toaster.create({
+  className: "recipe-toaster",
+  position: Position.TOP
+});
+
+/*
+A "user" error - eg, bad input
+*/
+export const postUserErrorToast = message =>
+  ErrorToastTopCenter.show({ message, intent: Intent.WARNING });
+
+/*
+A toast the user must dismiss manually, because they need to act on its information,
+ie., 8 bulk add genes out of 40 were bad. Manually see which ones and fix.
+*/
+export const keepAroundErrorToast = message =>
+  ErrorToastTopCenter.show({ message, timeout: 0, intent: Intent.WARNING });
+
+/*
+a hard network error
+*/
+export const postNetworkErrorToast = message =>
+  ErrorToastTopCenter.show({
+    message,
+    timeout: 30000,
+    intent: Intent.DANGER
+  });

client/src/components/geneExpression/cellSetButtons.js

@@ -0,0 +1,58 @@
+// jshint esversion: 6
+import React from "react";
+import { AnchorButton, Tooltip } from "@blueprintjs/core";
+import { connect } from "react-redux";
+import { World } from "../../util/stateManager";
+
+@connect()
+class CellSetButton extends React.Component {
+  set() {
+    const {
+      differential,
+      crossfilter,
+      dispatch,
+      eitherCellSetOneOrTwo
+    } = this.props;
+
+    // Reducer and components assume that value will be null if
+    // no selection made.  World..getSelectedByIndex() returns a
+    // zero length TypedArray when nothing is selected.
+    let set = World.getSelectedByIndex(crossfilter);
+    if (set.length === 0) set = null;
+
+    if (!differential.diffExp) {
+      /* diffexp needs to be cleared before we store a new set */
+      dispatch({
+        type: `store current cell selection as differential set ${eitherCellSetOneOrTwo}`,
+        data: set
+      });
+    }
+  }
+
+  render() {
+    const { differential, eitherCellSetOneOrTwo } = this.props;
+    const cellListName = `celllist${eitherCellSetOneOrTwo}`;
+    return (
+      <Tooltip
+        content="Save current selection for differential expression computation"
+        position="top"
+      >
+        <AnchorButton
+          style={{ marginRight: 10 }}
+          type="button"
+          disabled={differential.diffExp}
+          onClick={this.set.bind(this)}
+          data-testid={`cellset-button-${eitherCellSetOneOrTwo}`}
+        >
+          {eitherCellSetOneOrTwo}
+          {": "}
+          {differential[cellListName]
+            ? `${differential[cellListName].length} cells`
+            : "0 cells"}
+        </AnchorButton>
+      </Tooltip>
+    );
+  }
+}
+
+export default CellSetButton;

client/src/components/geneExpression/expressionButtons.js

@@ -0,0 +1,102 @@
+// jshint esversion: 6
+import React from "react";
+import _ from "lodash";
+import { Button, AnchorButton, Tooltip } from "@blueprintjs/core";
+import { connect } from "react-redux";
+import * as globals from "../../globals";
+import actions from "../../actions";
+import CellSetButton from "./cellSetButtons";
+
+@connect(state => ({
+  differential: state.differential,
+  world: state.controls.world,
+  crossfilter: state.controls.crossfilter,
+  selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null)
+}))
+class Expression extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = {};
+  }
+
+  computeDiffExp() {
+    const { dispatch, differential } = this.props;
+    if (differential.celllist1 && differential.celllist2) {
+      dispatch(
+        actions.requestDifferentialExpression(
+          differential.celllist1,
+          differential.celllist2
+        )
+      );
+    }
+  }
+
+  clearDifferentialExpression() {
+    const { dispatch, differential } = this.props;
+    dispatch({
+      type: "clear differential expression",
+      diffExp: differential.diffExp
+    });
+    dispatch({
+      type: "clear scatterplot"
+    });
+  }
+
+  render() {
+    const { differential } = this.props;
+    if (!differential) {
+      return null;
+    }
+    const haveBothCellSets =
+      !!differential.celllist1 && !!differential.celllist2;
+    return (
+      <div
+        style={{
+          marginRight: 10,
+          marginBottom: 10,
+          paddingLeft: globals.leftSidebarSectionPadding
+        }}
+      >
+        <CellSetButton {...this.props} eitherCellSetOneOrTwo={1} />
+        <CellSetButton {...this.props} eitherCellSetOneOrTwo={2} />
+        {!differential.diffExp ? (
+          <Tooltip
+            content="Add two cells selections, see the top 15 differentially expressed genes between them"
+            position="bottom"
+          >
+            <AnchorButton
+              style={{ marginTop: 10 }}
+              disabled={!haveBothCellSets}
+              intent="primary"
+              data-testid="diffexp-button"
+              loading={differential.loading}
+              fill
+              type="button"
+              onClick={this.computeDiffExp.bind(this)}
+            >
+              Compute Differential Expression
+            </AnchorButton>
+          </Tooltip>
+        ) : null}
+        {differential.diffExp ? (
+          <Tooltip
+            content="Remove differentially expressed gene list and clear cell selections"
+            position="bottom"
+          >
+            <Button
+              type="button"
+              fill
+              style={{ marginTop: 10 }}
+              intent="warning"
+              onClick={this.clearDifferentialExpression.bind(this)}
+            >
+              Clear Differential Expression
+            </Button>
+          </Tooltip>
+        ) : null}
+      </div>
+    );
+  }
+}
+
+export default Expression;

client/src/components/geneExpression/index.js

@@ -0,0 +1,347 @@
+// jshint esversion: 6
+/* rc slider https://www.npmjs.com/package/rc-slider */
+
+import React from "react";
+import _ from "lodash";
+import fuzzysort from "fuzzysort";
+
+import { connect } from "react-redux";
+import {
+  MenuItem,
+  Button,
+  FormGroup,
+  InputGroup,
+  ControlGroup
+} from "@blueprintjs/core";
+import { Suggest } from "@blueprintjs/select";
+import HistogramBrush from "../brushableHistogram";
+import * as globals from "../../globals";
+import actions from "../../actions";
+import {
+  postUserErrorToast,
+  keepAroundErrorToast
+} from "../framework/toasters";
+import ExpressionButtons from "./expressionButtons";
+import finiteExtent from "../../util/finiteExtent";
+
+const renderGene = (fuzzySortResult, { handleClick, modifiers, query }) => {
+  if (!modifiers.matchesPredicate) {
+    return null;
+  }
+  /* the fuzzysort wraps the object with other properties, like a score */
+  const geneName = fuzzySortResult.target;
+
+  return (
+    <MenuItem
+      active={modifiers.active}
+      disabled={modifiers.disabled}
+      // Use of annotations in this way is incorrect and dataset specific.
+      // See https://github.com/chanzuckerberg/cellxgene/issues/483
+      // label={gene.n_counts}
+      key={geneName}
+      onClick={g =>
+        /* this fires when user clicks a menu item */
+        handleClick(g)
+      }
+      text={geneName}
+    />
+  );
+};
+
+const filterGenes = (query, genes) =>
+  /* fires on load, once, and then for each character typed into the input */
+  fuzzysort.go(query, genes, {
+    limit: 5,
+    threshold: -10000 // don't return bad results
+  });
+
+@connect(state => {
+  return {
+    obsAnnotations: _.get(state.controls.world, "obsAnnotations", null),
+    userDefinedGenes: state.controls.userDefinedGenes,
+    userDefinedGenesLoading: state.controls.userDefinedGenesLoading,
+    world: state.controls.world,
+    colorAccessor: state.controls.colorAccessor,
+    differential: state.differential
+  };
+})
+class GeneExpression extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = {
+      bulkAdd: "",
+      tab: "autosuggest"
+    };
+  }
+
+  placeholderGeneNames() {
+    /*
+    return a string containing gene name suggestions for use as a user hint.
+    Eg.,    Apod, Cd74, ...
+    Will return a max of 3 genes, totalling 15 characters in length.
+    Randomly selects gene names.
+
+    NOTE: the random selection means it will re-render constantly.
+    */
+    const { world } = this.props;
+    const { varAnnotations } = world;
+    const geneNames = varAnnotations.col("name").asArray();
+    if (geneNames.length > 0) {
+      const placeholder = [];
+      let len = geneNames.length;
+      const maxGeneNameCount = 3;
+      const maxStrLength = 15;
+      len = len < maxGeneNameCount ? len : maxGeneNameCount;
+      for (let i = 0, strLen = 0; i < len && strLen < maxStrLength; i += 1) {
+        const deal = Math.floor(Math.random() * geneNames.length);
+        const geneName = geneNames[deal];
+        placeholder.push(geneName);
+        strLen += geneName.length + 2; // '2' is the length of a comma and space
+      }
+      placeholder.push("...");
+      return placeholder.join(", ");
+    }
+    // default - should never happen.
+    return "Apod, Cd74, ...";
+  }
+
+  handleClick(g) {
+    const { world, dispatch, userDefinedGenes } = this.props;
+    const gene = g.target;
+    if (userDefinedGenes.indexOf(gene) !== -1) {
+      postUserErrorToast("That gene already exists");
+    } else if (userDefinedGenes.length > 15) {
+      postUserErrorToast(
+        "That's too many genes, you can have at most 15 user defined genes"
+      );
+    } else if (world.varAnnotations.col("name").indexOf(gene) === undefined) {
+      postUserErrorToast("That doesn't appear to be a valid gene name.");
+    } else {
+      dispatch(actions.requestUserDefinedGene(gene));
+      dispatch({
+        type: "user defined gene",
+        data: gene
+      });
+    }
+  }
+
+  handleBulkAddClick() {
+    const { world, dispatch, userDefinedGenes } = this.props;
+    const { bulkAdd } = this.state;
+
+    /*
+      test:
+      Apod,,, Cd74,,    ,,,    Foo,    Bar-2,,
+    */
+    if (bulkAdd !== "") {
+      const genes = _.pull(_.uniq(bulkAdd.split(/[ ,]+/)), "");
+
+      genes.forEach(gene => {
+        if (gene.length === 0) {
+          keepAroundErrorToast("Must enter a gene name.");
+        } else if (userDefinedGenes.indexOf(gene) !== -1) {
+          keepAroundErrorToast("That gene already exists");
+        } else if (
+          world.varAnnotations.col("name").indexOf(gene) === undefined
+        ) {
+          keepAroundErrorToast(
+            `${gene} doesn't appear to be a valid gene name.`
+          );
+        } else {
+          dispatch(actions.requestUserDefinedGene(gene));
+          dispatch({
+            type: "user defined gene",
+            data: gene
+          });
+        }
+      });
+    }
+
+    this.setState({ bulkAdd: "" });
+  }
+
+  render() {
+    const {
+      world,
+      userDefinedGenes,
+      userDefinedGenesLoading,
+      differential
+    } = this.props;
+
+    const { tab, bulkAdd } = this.state;
+
+    return (
+      <div>
+        <div
+          style={{
+            marginTop: 30
+          }}
+        >
+          <p
+            style={Object.assign({}, globals.leftSidebarSectionHeading, {
+              paddingLeft: globals.leftSidebarSectionPadding,
+              margin: 0
+            })}
+          >
+            Selected Genes
+          </p>
+          <div
+            style={{
+              padding: globals.leftSidebarSectionPadding
+            }}
+          >
+            <Button
+              active={tab === "autosuggest"}
+              style={{ marginRight: 5 }}
+              minimal
+              small
+              onClick={() => {
+                this.setState({ tab: "autosuggest" });
+              }}
+            >
+              Autosuggest
+            </Button>
+            <Button
+              active={tab === "bulkadd"}
+              minimal
+              small
+              onClick={() => {
+                this.setState({ tab: "bulkadd" });
+              }}
+            >
+              Bulk add genes
+            </Button>
+          </div>
+
+          {tab === "autosuggest" ? (
+            <ControlGroup
+              style={{
+                paddingLeft: globals.leftSidebarSectionPadding,
+                paddingBottom: globals.leftSidebarSectionPadding
+              }}
+            >
+              <Suggest
+                closeOnSelect
+                openOnKeyDown
+                resetOnSelect
+                itemDisabled={
+                  userDefinedGenesLoading ? () => true : () => false
+                }
+                noResults={<MenuItem disabled text="No matching genes." />}
+                onItemSelect={g => {
+                  /* this happens on 'enter' */
+                  this.handleClick(g);
+                }}
+                inputProps={{ "data-testid": "gene-search" }}
+                inputValueRenderer={g => {
+                  return "";
+                }}
+                itemListPredicate={filterGenes}
+                itemRenderer={renderGene.bind(this)}
+                items={
+                  world && world.varAnnotations
+                    ? world.varAnnotations.col("name").asArray()
+                    : ["No genes"]
+                }
+                popoverProps={{ minimal: true }}
+              />
+              <Button
+                className="bp3-button bp3-intent-primary"
+                data-testid={"add-gene"}
+                loading={userDefinedGenesLoading}
+              >
+                Add
+              </Button>
+            </ControlGroup>
+          ) : null}
+          {tab === "bulkadd" ? (
+            <div style={{ paddingLeft: globals.leftSidebarSectionPadding }}>
+              <form
+                onSubmit={e => {
+                  e.preventDefault();
+                  this.handleBulkAddClick();
+                }}
+              >
+                <FormGroup
+                  helperText="Add a list of genes (comma delimited)"
+                  labelFor="text-input-bulk-add"
+                >
+                  <ControlGroup>
+                    <InputGroup
+                      onChange={e => {
+                        this.setState({ bulkAdd: e.target.value });
+                      }}
+                      id="text-input-bulk-add"
+                      placeholder={this.placeholderGeneNames()}
+                      value={bulkAdd}
+                    />
+                    <Button
+                      intent="primary"
+                      onClick={this.handleBulkAddClick.bind(this)}
+                      loading={userDefinedGenesLoading}
+                    >
+                      Add
+                    </Button>
+                  </ControlGroup>
+                </FormGroup>
+              </form>
+            </div>
+          ) : null}
+          {world && userDefinedGenes.length > 0
+            ? _.map(userDefinedGenes, (geneName, index) => {
+                const values = world.varData.col(geneName);
+                if (!values) {
+                  return null;
+                }
+                const summary = values.summarize();
+                return (
+                  <HistogramBrush
+                    key={geneName}
+                    field={geneName}
+                    zebra={index % 2 === 0}
+                    ranges={summary}
+                    isUserDefined
+                  />
+                );
+              })
+            : null}
+        </div>
+        <div>
+          <p
+            style={Object.assign({}, globals.leftSidebarSectionHeading, {
+              marginTop: 40,
+              paddingLeft: globals.leftSidebarSectionPadding
+            })}
+          >
+            Differentially Expressed Genes
+          </p>
+          <ExpressionButtons />
+          {differential.diffExp
+            ? _.map(differential.diffExp, (value, index) => {
+                const name = world.varAnnotations.at(value[0], "name");
+                const values = world.varData.col(name);
+                if (!values) {
+                  return null;
+                }
+                const summary = values.summarize();
+                return (
+                  <HistogramBrush
+                    key={name}
+                    field={name}
+                    zebra={index % 2 === 0}
+                    ranges={summary}
+                    isDiffExp
+                    logFoldChange={value[1]}
+                    pval={value[2]}
+                    pvalAdj={value[3]}
+                  />
+                );
+              })
+            : null}
+        </div>
+      </div>
+    );
+  }
+}
+
+export default GeneExpression;

client/src/components/graph/drawPointsRegl.js

@@ -38,15 +38,7 @@ export default function(regl) {
     uniforms: {
       distance: regl.prop("distance"),
       view: regl.prop("view"),
-      projection: (context, props) => {
+      projection: ({viewportWidth, viewportHeight}) => mat4.perspective([], Math.PI / 2, viewportWidth / viewportHeight, 0.01, 1000)
-        return mat4.perspective(
-          [],
-          Math.PI / 2,
-          context.viewportWidth * props.scale / context.viewportHeight,
-          0.01,
-          1000
-        );
-      }
     },
 
     count: regl.prop("count"),

client/src/components/graph/graph.css

@@ -1,4 +1,4 @@
-.graphCanvas path {
+:local(.graphCanvas) path {
   shape-rendering: crispEdges;
 }
 
@@ -16,8 +16,8 @@
   pointer-events: none;
 }
 
-.axis path,
+:local(.axis) path,
-.axis line {
+:local(.axis) line {
   fill: none;
   stroke: #000;
   stroke-width: 1px;
@@ -29,6 +29,6 @@ circle {
   fill: none;
 }
 
-.hidden {
+:local(.hidden) {
   display: none;
 }

client/src/components/graph/graph.js

@@ -0,0 +1,492 @@
+// jshint esversion: 6
+import React from "react";
+import _ from "lodash";
+import * as d3 from "d3";
+import { connect } from "react-redux";
+import mat4 from "gl-mat4";
+import _regl from "regl";
+import { Button, AnchorButton, Tooltip } from "@blueprintjs/core";
+import * as globals from "../../globals";
+import setupSVGandBrushElements from "./setupSVGandBrush";
+import actions from "../../actions";
+import _camera from "../../util/camera";
+import _drawPoints from "./drawPointsRegl";
+import scaleLinear from "../../util/scaleLinear";
+
+/* https://bl.ocks.org/mbostock/9078690 - quadtree for onClick / hover selections */
+
+@connect(state => ({
+  world: state.controls.world,
+  universe: state.controls.universe,
+  crossfilter: state.controls.crossfilter,
+  responsive: state.responsive,
+  colorRGB: _.get(state.controls, "colorRGB", null),
+  opacityForDeselectedCells: state.controls.opacityForDeselectedCells,
+  selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null),
+  resettingInterface: state.controls.resettingInterface
+}))
+class Graph extends React.Component {
+  constructor(props) {
+    super(props);
+    this.count = 0;
+    this.inverse = mat4.identity([]);
+    this.graphPaddingTop = 0;
+    this.graphPaddingBottom = 45;
+    this.graphPaddingRight = globals.leftSidebarWidth;
+    this.renderCache = {
+      positions: null,
+      colors: null,
+      sizes: null
+    };
+    this.state = {
+      svg: null,
+      brush: null,
+      mode: "lasso"
+    };
+  }
+
+  componentDidMount() {
+    // setup canvas and camera
+    const camera = _camera(this.reglCanvas, { scale: true, rotate: false });
+    const regl = _regl(this.reglCanvas);
+
+    const drawPoints = _drawPoints(regl);
+
+    // preallocate buffers
+    const pointBuffer = regl.buffer();
+    const colorBuffer = regl.buffer();
+    const sizeBuffer = regl.buffer();
+
+    /* first time, but this duplicates above function, should be possile to avoid this */
+    const reglRender = regl.frame(() => {
+      this.reglDraw(
+        regl,
+        drawPoints,
+        sizeBuffer,
+        colorBuffer,
+        pointBuffer,
+        camera
+      );
+      camera.tick();
+    });
+
+    this.reglRenderState = "rendering";
+
+    this.setState({
+      regl,
+      drawPoints,
+      pointBuffer,
+      colorBuffer,
+      sizeBuffer,
+      camera,
+      reglRender
+    });
+  }
+
+  componentDidUpdate(prevProps) {
+    const { renderCache } = this;
+    const {
+      world,
+      crossfilter,
+      colorRGB,
+      responsive,
+      selectionUpdate
+    } = this.props;
+    const {
+      reglRender,
+      mode,
+      regl,
+      drawPoints,
+      camera,
+      pointBuffer,
+      colorBuffer,
+      sizeBuffer,
+      svg
+    } = this.state;
+
+    if (reglRender && this.reglRenderState === "rendering" && mode !== "zoom") {
+      reglRender.cancel();
+      this.reglRenderState = "paused";
+    }
+
+    if (regl && world) {
+      /* update the regl state */
+      const { obsLayout, nObs } = world;
+      const X = obsLayout.col("X").asArray();
+      const Y = obsLayout.col("Y").asArray();
+
+      // X/Y positions for each point - a cached value that only
+      // changes if we have loaded entirely new cell data
+      //
+      if (!renderCache.positions || world !== prevProps.world) {
+        renderCache.positions = new Float32Array(2 * nObs);
+
+        const glScaleX = scaleLinear([0, 1], [-1, 1]);
+        const glScaleY = scaleLinear([0, 1], [1, -1]);
+
+        const offset = [d3.mean(X) - 0.5, d3.mean(Y) - 0.5];
+
+        for (let i = 0, { positions } = renderCache; i < nObs; i += 1) {
+          positions[2 * i] = glScaleX(X[i] - offset[0]);
+          positions[2 * i + 1] = glScaleY(Y[i] - offset[1]);
+        }
+        pointBuffer({
+          data: renderCache.positions,
+          dimension: 2
+        });
+
+        this.setState({
+          offset
+        });
+      }
+
+      // Colors for each point - a cached value that only changes when
+      // the cell metadata changes (done by updateCellColors middleware).
+      // NOTE: this is a slightly pessimistic assumption, as the metadata
+      // could have changed for some other reason, but for now color is
+      // the only metadata that changes client-side.  If this is problematic,
+      // we could add some sort of color-specific indicator to the app state.
+      if (!renderCache.colors || colorRGB !== prevProps.colorRGB) {
+        const rgb = colorRGB;
+        if (!renderCache.colors) {
+          renderCache.colors = new Float32Array(3 * rgb.length);
+        }
+        for (let i = 0, { colors } = renderCache; i < rgb.length; i += 1) {
+          colors.set(rgb[i], 3 * i);
+        }
+        colorBuffer({ data: renderCache.colors, dimension: 3 });
+      }
+
+      // Sizes for each point - updates are triggered only when selected
+      // obs change
+      if (!renderCache.sizes || selectionUpdate !== prevProps.selectionUpdate) {
+        if (!renderCache.sizes) {
+          renderCache.sizes = new Float32Array(nObs);
+        }
+        crossfilter.fillByIsFiltered(renderCache.sizes, 4, 0.2);
+        sizeBuffer({ data: renderCache.sizes, dimension: 1 });
+      }
+
+      this.count = nObs;
+
+      regl._refresh();
+      this.reglDraw(
+        regl,
+        drawPoints,
+        sizeBuffer,
+        colorBuffer,
+        pointBuffer,
+        camera
+      );
+    }
+
+    if (
+      prevProps.responsive.height !== responsive.height ||
+      prevProps.responsive.width !== responsive.width ||
+      /* first time */
+      (responsive.height && responsive.width && !svg)
+    ) {
+      /* clear out whatever was on the div, even if nothing, but usually the brushes etc */
+      d3.select("#graphAttachPoint")
+        .selectAll("svg")
+        .remove();
+      const { svg: newSvg, brush } = setupSVGandBrushElements(
+        this.handleBrushSelectAction.bind(this),
+        this.handleBrushDeselectAction.bind(this),
+        responsive,
+        this.graphPaddingRight,
+        this.handleLassoStart.bind(this),
+        this.handleLassoEnd.bind(this)
+      );
+      this.setState({ svg: newSvg, brush });
+    }
+  }
+
+  reglDraw(regl, drawPoints, sizeBuffer, colorBuffer, pointBuffer, camera) {
+    regl.clear({
+      depth: 1,
+      color: [1, 1, 1, 1]
+    });
+    drawPoints({
+      size: sizeBuffer,
+      distance: camera.distance,
+      color: colorBuffer,
+      position: pointBuffer,
+      count: this.count,
+      view: camera.view()
+    });
+  }
+
+  restartReglLoop() {
+    const {
+      regl,
+      drawPoints,
+      sizeBuffer,
+      colorBuffer,
+      pointBuffer,
+      camera
+    } = this.state;
+    const reglRender = regl.frame(() => {
+      this.reglDraw(
+        regl,
+        drawPoints,
+        sizeBuffer,
+        colorBuffer,
+        pointBuffer,
+        camera
+      );
+      camera.tick();
+    });
+
+    this.reglRenderState = "rendering";
+
+    this.setState({
+      reglRender
+    });
+  }
+
+  invertPoint(pin) {
+    const { responsive } = this.props;
+    const { regl, camera, offset } = this.state;
+
+    const gl = regl._gl;
+
+    // get aspect ratio
+    const aspect = gl.drawingBufferWidth / gl.drawingBufferHeight;
+
+    // compute inverse view matrix
+    const inverse = mat4.invert([], camera.view());
+
+    // transform screen coordinates -> cell coordinates
+    const x = (2 * pin[0]) / (responsive.width - this.graphPaddingRight) - 1;
+    const y = 2 * (1 - pin[1] / (responsive.height - this.graphPaddingTop)) - 1;
+    const pout = [
+      x * inverse[14] * aspect + inverse[12],
+      y * inverse[14] + inverse[13]
+    ];
+    return [(pout[0] + 1) / 2 + offset[0], (pout[1] + 1) / 2 + offset[1]];
+  }
+
+  handleBrushSelectAction() {
+    /*
+      This conditional handles procedural brush deselect. Brush emits
+      an event on procedural deselect because it is move: null
+    */
+    /*
+      event describing brush position:
+      @-------|
+      |       |
+      |       |
+      |-------@
+    */
+    /*
+      No idea why d3 event scope works like this
+      but apparently
+      it does
+      https://bl.ocks.org/EfratVil/0e542f5fc426065dd1d4b6daaa345a9f
+    */
+    const { dispatch } = this.props;
+
+    if (d3.event.sourceEvent !== null) {
+      const s = d3.event.selection;
+
+      const brushCoords = {
+        northwest: this.invertPoint([s[0][0], s[0][1]]),
+        southeast: this.invertPoint([s[1][0], s[1][1]])
+      };
+
+      dispatch({
+        type: "graph brush selection change",
+        brushCoords
+      });
+    }
+  }
+
+  handleBrushDeselectAction() {
+    const { dispatch } = this.props;
+    const { svg, brush } = this.state;
+
+    if (d3.event && !d3.event.selection) {
+      dispatch({
+        type: "graph brush deselect"
+      });
+    }
+
+    if (!d3.event) {
+      /*
+      this line clears the brush procedurally, ie., zoom button clicked,
+      not a click away from brush on svg
+      */
+      svg.select(".graph_brush").call(brush.move, null);
+      dispatch({
+        type: "graph brush deselect"
+      });
+    }
+  }
+
+  handleLassoStart() {
+    const { dispatch } = this.props;
+    // reset selected points when starting a new polygon
+    // making it easier for the user to make the next selection
+    dispatch({
+      type: "lasso started"
+    });
+  }
+
+  // when a lasso is completed, filter to the points within the lasso polygon
+  handleLassoEnd(polygon) {
+    const { dispatch } = this.props;
+
+    dispatch({
+      type: "lasso selection",
+      polygon: polygon.map(xy => this.invertPoint(xy)) // transform the polygon
+    });
+  }
+
+  handleOpacityRangeChange(e) {
+    const { dispatch } = this.props;
+    dispatch({
+      type: "change opacity deselected cells in 2d graph background",
+      data: e.target.value
+    });
+  }
+
+  resetInterface() {
+    const { dispatch } = this.props;
+    dispatch({
+      type: "interface reset started"
+    });
+    dispatch(actions.resetInterface());
+  }
+
+  render() {
+    const {
+      dispatch,
+      responsive,
+      crossfilter,
+      resettingInterface
+    } = this.props;
+
+    const { mode } = this.state;
+    return (
+      <div id="graphWrapper">
+        <div
+          style={{
+            position: "fixed",
+            right: 0,
+            top: 0
+          }}
+        >
+          <div
+            style={{
+              padding: 10,
+              display: "flex",
+              justifyContent: "flex-end",
+              alignItems: "baseline"
+            }}
+          >
+            <Tooltip
+              content="Show only metadata and cells which are currently selected"
+              position="left"
+            >
+              <AnchorButton
+                type="button"
+                disabled={
+                  crossfilter &&
+                  (crossfilter.countFiltered() === 0 ||
+                    crossfilter.countFiltered() === crossfilter.size())
+                }
+                style={{ marginRight: 10 }}
+                onClick={() => {
+                  dispatch(actions.regraph());
+                  dispatch({ type: "increment graph render counter" });
+                }}
+              >
+                subset to current selection
+              </AnchorButton>
+            </Tooltip>
+            <Tooltip
+              content="Reset cellxgene, clearing all selections"
+              position="left"
+            >
+              <AnchorButton
+                disabled={
+                  false
+                  /* world && universe ? worldEqUniverse(world, universe) : false */
+                }
+                type="button"
+                loading={resettingInterface}
+                intent="warning"
+                style={{ marginRight: 10 }}
+                onClick={this.resetInterface.bind(this)}
+              >
+                reset
+              </AnchorButton>
+            </Tooltip>
+            <div>
+              <div className="bp3-button-group">
+                <Tooltip content="Lasso selection" position="left">
+                  <Button
+                    type="button"
+                    className="bp3-button bp3-icon-polygon-filter"
+                    active={mode === "lasso"}
+                    onClick={() => {
+                      this.handleBrushDeselectAction();
+                      // this.restartReglLoop();
+                      this.setState({ mode: "lasso" });
+                    }}
+                    style={{
+                      cursor: "pointer"
+                    }}
+                  />
+                </Tooltip>
+                <Tooltip content="Pan and zoom" position="left">
+                  <Button
+                    type="button"
+                    className="bp3-button bp3-icon-zoom-in"
+                    active={mode === "zoom"}
+                    onClick={() => {
+                      this.handleBrushDeselectAction();
+                      this.restartReglLoop();
+                      this.setState({ mode: "zoom" });
+                    }}
+                    style={{
+                      cursor: "pointer"
+                    }}
+                  />
+                </Tooltip>
+              </div>
+            </div>
+          </div>
+        </div>
+        <div
+          style={{
+            zIndex: -9999,
+            position: "fixed",
+            top: 0,
+            right: 0
+          }}
+        >
+          <div
+            style={{
+              display: mode === "lasso" ? "inherit" : "none"
+            }}
+            id="graphAttachPoint"
+          />
+          <div style={{ padding: 0, margin: 0 }}>
+            <canvas
+              width={responsive.width - this.graphPaddingRight}
+              height={responsive.height - this.graphPaddingTop}
+              data-testid="layout"
+              ref={canvas => {
+                this.reglCanvas = canvas;
+              }}
+            />
+          </div>
+        </div>
+      </div>
+    );
+  }
+}
+
+export default Graph;

client/src/components/graph/setupLasso.js

@@ -0,0 +1,127 @@
+// https://bl.ocks.org/pbeshai/8008075f9ce771ee8be39e8c38907570
+
+import * as d3 from "d3";
+
+const Lasso = () => {
+  const dispatch = d3.dispatch("start", "end");
+
+  const polygonToPath = polygon =>
+    `M${polygon.map(d => d.join(",")).join("L")}`;
+
+  const distance = (pt1, pt2) =>
+    Math.sqrt((pt2[0] - pt1[0]) ** 2 + (pt2[1] - pt1[1]) ** 2);
+
+  // distance last point has to be to first point before it auto closes when mouse is released
+  const closeDistance = 75;
+
+  const lasso = svg => {
+    let lassoPolygon;
+    let lassoPath;
+    let closePath;
+
+    const handleDragStart = () => {
+      lassoPolygon = [d3.mouse(svg.node())]; // current x y of mouse within element
+
+      if (lassoPath) {
+        lassoPath.remove();
+      }
+
+      lassoPath = g
+        .append("path")
+        .attr("fill", "#0bb")
+        .attr("fill-opacity", 0.1)
+        .attr("stroke", "#0bb")
+        .attr("stroke-dasharray", "3, 3");
+
+      closePath = g
+        .append("line")
+        .attr("x2", lassoPolygon[0][0])
+        .attr("y2", lassoPolygon[0][1])
+        .attr("stroke", "#0bb")
+        .attr("stroke-dasharray", "3, 3")
+        .attr("opacity", 0);
+
+      dispatch.call("start", lasso, lassoPolygon);
+    };
+
+    const handleDrag = () => {
+      const point = d3.mouse(svg.node());
+      lassoPolygon.push(point);
+      lassoPath.attr("d", polygonToPath(lassoPolygon));
+
+      // indicate if we are within closing distance
+      if (
+        distance(lassoPolygon[0], lassoPolygon[lassoPolygon.length - 1]) <
+        closeDistance
+      ) {
+        closePath
+          .attr("x1", point[0])
+          .attr("y1", point[1])
+          .attr("opacity", 1);
+      } else {
+        closePath.attr("opacity", 0);
+      }
+    };
+
+    const handleDragEnd = () => {
+      // remove the close path
+      closePath.remove();
+      closePath = null;
+
+      // succesfully closed
+      if (
+        distance(lassoPolygon[0], lassoPolygon[lassoPolygon.length - 1]) <
+        closeDistance
+      ) {
+        lassoPath.attr("d", `${polygonToPath(lassoPolygon)}Z`);
+        dispatch.call("end", lasso, lassoPolygon);
+
+        // otherwise cancel
+      } else {
+        lassoPath.remove();
+        lassoPath = null;
+        lassoPolygon = null;
+      }
+    };
+
+    // append a <g> with a rect
+    const g = svg.append("g").attr("class", "lasso-group");
+    const bbox = svg.node().getBoundingClientRect();
+    const area = g
+      .append("rect")
+      .attr("width", bbox.width)
+      .attr("height", bbox.height)
+      .attr("fill", "tomato")
+      .attr("opacity", 0);
+
+    const drag = d3
+      .drag()
+      .on("start", handleDragStart)
+      .on("drag", handleDrag)
+      .on("end", handleDragEnd);
+
+    area.call(drag);
+
+    lasso.reset = () => {
+      if (lassoPath) {
+        lassoPath.remove();
+        lassoPath = null;
+      }
+
+      lassoPolygon = null;
+      if (closePath) {
+        closePath.remove();
+        closePath = null;
+      }
+    };
+  };
+
+  lasso.on = (type, callback) => {
+    dispatch.on(type, callback);
+    return lasso;
+  };
+
+  return lasso;
+};
+
+export default Lasso;

client/src/components/graph/setupSVGandBrush.js

@@ -0,0 +1,51 @@
+// jshint esversion: 6
+import * as d3 from "d3";
+import styles from "./graph.css";
+import Lasso from "./setupLasso";
+
+/******************************************
+*******************************************
+          put svg & brush in DOM
+*******************************************
+******************************************/
+
+export default (
+  handleBrushSelectAction,
+  handleBrushDeselectAction,
+  responsive,
+  graphPaddingRight,
+  handleLassoStart,
+  handleLassoEnd
+) => {
+  const svg = d3
+    .select("#graphAttachPoint")
+    .append("svg")
+    .attr("data-testid", "layout-overlay")
+    .attr("width", responsive.width - graphPaddingRight)
+    .attr("height", responsive.height)
+    .attr("class", `${styles.graphSVG}`);
+
+  const brush = d3
+    .brush()
+    .extent([[0, 0], [responsive.width - graphPaddingRight, responsive.height]])
+    .on("brush", handleBrushSelectAction)
+    .on("end", handleBrushDeselectAction);
+
+  const brushContainer = svg
+    .append("g")
+    .attr("class", "graph_brush")
+    .call(brush);
+
+  const lassoInstance = Lasso()
+    .on("end", handleLassoEnd)
+    .on("start", handleLassoStart);
+
+  const lasso = svg.call(lassoInstance);
+
+  return {
+    svg,
+    brushContainer,
+    brush,
+    lasso
+  };
+};

client/src/components/leftsidebar.js

@@ -0,0 +1,77 @@
+// jshint esversion: 6
+import _ from "lodash";
+import React from "react";
+import { connect } from "react-redux";
+import Categorical from "./categorical/categorical";
+import Continuous from "./continuous/continuous";
+import GeneExpression from "./geneExpression";
+import * as globals from "../globals";
+import DynamicScatterplot from "./scatterplot/scatterplot";
+
+@connect(state => ({
+  responsive: state.responsive,
+  datasetTitle: _.get(state.config, "displayNames.dataset"),
+  scatterplotXXaccessor: state.controls.scatterplotXXaccessor,
+  scatterplotYYaccessor: state.controls.scatterplotYYaccessor
+}))
+class LeftSideBar extends React.Component {
+  render() {
+    const {
+      responsive,
+      datasetTitle,
+      scatterplotXXaccessor,
+      scatterplotYYaccessor
+    } = this.props;
+
+    /*
+    this magic number should be made less fragile,
+    if cellxgene logo or tabs change, this must as well
+    */
+    const metadataSectionPadding = 0;
+    // scatterplotXXaccessor && scatterplotYYaccessor ? 450 : 0;
+
+    return (
+      <div
+        style={{
+          position: "fixed",
+          backgroundColor: "white",
+          /* x y blur spread color */
+          boxShadow: "1px 0px 6px 2px rgba(153,153,153,0.4)"
+        }}
+      >
+        <p
+          data-testid="header"
+          style={{
+            position: "fixed",
+            top: globals.cellxgeneTitleTopPadding,
+            left: globals.leftSidebarWidth + globals.cellxgeneTitleLeftPadding,
+            margin: 0,
+            fontSize: globals.largestFontSize,
+            color: globals.darkerGrey,
+            width: "100%"
+          }}
+        >
+          cellxgene: {datasetTitle}
+        </p>
+
+        <div
+          style={{
+            height: responsive.height - metadataSectionPadding,
+            width: globals.leftSidebarWidth,
+            overflowY: "auto",
+            overflowX: "hidden"
+          }}
+        >
+          <Categorical />
+          <GeneExpression />
+          <Continuous />
+        </div>
+        {scatterplotXXaccessor && scatterplotYYaccessor ? (
+          <DynamicScatterplot />
+        ) : null}
+      </div>
+    );
+  }
+}
+
+export default LeftSideBar;

client/src/components/scatterplot/drawPointsRegl.js

@@ -38,15 +38,7 @@ export default function(regl) {
     uniforms: {
       distance: regl.prop("distance"),
       view: regl.prop("view"),
-      projection: (context, props) => {
+      projection: () => mat4.perspective([], Math.PI / 2, 1, 0.01, 1000)
-        return mat4.perspective(
-          [],
-          Math.PI / 2,
-          context.viewportWidth * props.scale / context.viewportHeight,
-          0.01,
-          1000
-        );
-      }
     },
 
     count: regl.prop("count"),

client/src/components/scatterplot/scatterplot.css

@@ -1,15 +1,14 @@
-
+:local(.scatterplot) {
-.scatterplot {
   display: block;
 }
 
-.scatterplot svg,
+:local(.scatterplot) svg,
-.scatterplot canvas {
+:local(.scatterplot) canvas {
   font: 10px sans-serif;
   position: absolute;
 }
 
-.scatterplot canvas {
+:local(.scatterplot) canvas {
   opacity: 0.9;
   pointer-events: none;
 }

client/src/components/scatterplot/scatterplot.js

@@ -0,0 +1,385 @@
+// jshint esversion: 6
+// https://bl.ocks.org/Jverma/076377dd0125b1a508621441752735fc
+// https://peterbeshai.com/scatterplot-in-d3-with-voronoi-interaction.html
+
+import React from "react";
+import _ from "lodash";
+import { connect } from "react-redux";
+import { Button, ButtonGroup } from "@blueprintjs/core";
+import _regl from "regl";
+import * as d3 from "d3";
+import * as globals from "../../globals";
+
+import _camera from "../../util/camera";
+
+import setupScatterplot from "./setupScatterplot";
+import styles from "./scatterplot.css";
+
+import _drawPoints from "./drawPointsRegl";
+import scaleLinear from "../../util/scaleLinear";
+
+import { margin, width, height } from "./util";
+import finiteExtent from "../../util/finiteExtent";
+
+@connect(state => {
+  const {
+    world,
+    crossfilter,
+    scatterplotXXaccessor,
+    scatterplotYYaccessor
+  } = state.controls;
+  const expressionX =
+    world &&
+    scatterplotXXaccessor &&
+    world.varData.hasCol(scatterplotXXaccessor)
+      ? world.varData.col(scatterplotXXaccessor).asArray()
+      : null;
+  const expressionY =
+    world &&
+    scatterplotYYaccessor &&
+    world.varData.hasCol(scatterplotYYaccessor)
+      ? world.varData.col(scatterplotYYaccessor).asArray()
+      : null;
+
+  return {
+    world,
+
+    colorRGB: state.controls.colorRGB,
+    colorAccessor: state.controls.colorAccessor,
+    colorScale: state.controls.colorScale,
+
+    // Accessors are var/gene names (strings)
+    scatterplotXXaccessor,
+    scatterplotYYaccessor,
+    opacityForDeselectedCells: state.controls.opacityForDeselectedCells,
+
+    differential: state.differential,
+
+    expressionX,
+    expressionY,
+
+    crossfilter,
+    // updated whenever the crossfilter selection is updated
+    selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null)
+  };
+})
+class Scatterplot extends React.Component {
+  constructor(props) {
+    super(props);
+    this.count = 0;
+    this.axes = false;
+    this.renderCache = {
+      positions: null,
+      colors: null,
+      sizes: null,
+      xScale: null,
+      yScale: null
+    };
+    this.state = {
+      svg: null,
+      minimized: null
+    };
+  }
+
+  componentDidMount() {
+    const { svg } = setupScatterplot(width, height, margin);
+    let scales;
+    const { expressionX, expressionY } = this.props;
+
+    if (svg && expressionX && expressionY) {
+      scales = Scatterplot.setupScales(expressionX, expressionY);
+      this.drawAxesSVG(scales.xScale, scales.yScale, svg);
+      this.renderCache = { ...this.renderCache, ...scales };
+    }
+
+    const camera = _camera(this.reglCanvas, { scale: true, rotate: false });
+    const regl = _regl(this.reglCanvas);
+
+    const drawPoints = _drawPoints(regl);
+
+    // preallocate buffers
+    const pointBuffer = regl.buffer();
+    const colorBuffer = regl.buffer();
+    const sizeBuffer = regl.buffer();
+
+    const reglRender = regl.frame(() => {
+      this.reglDraw(
+        regl,
+        drawPoints,
+        sizeBuffer,
+        colorBuffer,
+        pointBuffer,
+        camera
+      );
+      camera.tick();
+    });
+
+    this.reglRenderState = "rendering";
+
+    this.setState({
+      regl,
+      sizeBuffer,
+      pointBuffer,
+      colorBuffer,
+      svg,
+      reglRender,
+      camera,
+      drawPoints
+    });
+  }
+
+  componentDidUpdate(prevProps) {
+    const {
+      world,
+      crossfilter,
+      scatterplotXXaccessor,
+      scatterplotYYaccessor,
+      expressionX,
+      expressionY,
+      colorRGB,
+      selectionUpdate
+    } = this.props;
+    const {
+      reglRender,
+      regl,
+      pointBuffer,
+      colorBuffer,
+      sizeBuffer,
+      svg,
+      drawPoints,
+      camera
+    } = this.state;
+
+    if (
+      scatterplotXXaccessor !== prevProps.scatterplotXXaccessor || // was CLU now FTH1 etc
+      scatterplotYYaccessor !== prevProps.scatterplotYYaccessor // was CLU now FTH1 etc
+    ) {
+      const scales = Scatterplot.setupScales(expressionX, expressionY);
+      this.drawAxesSVG(scales.xScale, scales.yScale, svg);
+      this.renderCache = { ...this.renderCache, ...scales };
+    }
+
+    if (reglRender && this.reglRenderState === "rendering") {
+      reglRender.cancel();
+      this.reglRenderState = "paused";
+    }
+
+    if (
+      world &&
+      regl &&
+      pointBuffer &&
+      colorBuffer &&
+      sizeBuffer &&
+      expressionX &&
+      expressionY &&
+      scatterplotXXaccessor &&
+      scatterplotYYaccessor
+    ) {
+      const { renderCache } = this;
+      const { xScale, yScale } = this.renderCache;
+      const cellCount = expressionX.length;
+
+      // Points change when expressionX or expressionY change.
+      if (
+        !renderCache.positions ||
+        expressionX !== prevProps.expressionX ||
+        expressionY !== prevProps.expressionY
+      ) {
+        if (!renderCache.positions) {
+          renderCache.positions = new Float32Array(2 * cellCount);
+        }
+        const glScaleX = scaleLinear([0, width], [-0.95, 0.95]);
+        const glScaleY = scaleLinear([0, height], [-1, 1]);
+        for (let i = 0, { positions } = renderCache; i < cellCount; i += 1) {
+          positions[2 * i] = glScaleX(xScale(expressionX[i]));
+          positions[2 * i + 1] = glScaleY(yScale(expressionY[i]));
+        }
+        pointBuffer({ data: renderCache.positions, dimension: 2 });
+      }
+
+      // Colors for each point - change only when props.colorsRGB change.
+      if (!renderCache.colors || colorRGB !== prevProps.colorRGB) {
+        if (!renderCache.colors) {
+          renderCache.colors = new Float32Array(3 * cellCount);
+        }
+        for (let i = 0, { colors } = renderCache; i < cellCount; i += 1) {
+          colors.set(colorRGB[i], 3 * i);
+        }
+        colorBuffer({ data: renderCache.colors, dimension: 3 });
+      }
+
+      // Sizes for each point - updates are triggered only when selected
+      // obs change
+      if (!renderCache.sizes || selectionUpdate !== prevProps.selctionUpdate) {
+        if (!renderCache.sizes) {
+          renderCache.sizes = new Float32Array(cellCount);
+        }
+        crossfilter.fillByIsFiltered(renderCache.sizes, 4, 0.2);
+        sizeBuffer({ data: renderCache.sizes, dimension: 1 });
+      }
+
+      this.count = cellCount;
+
+      regl._refresh();
+      this.reglDraw(
+        regl,
+        drawPoints,
+        sizeBuffer,
+        colorBuffer,
+        pointBuffer,
+        camera
+      );
+    }
+  }
+
+  static setupScales(expressionX, expressionY) {
+    const xScale = d3
+      .scaleLinear()
+      .domain(finiteExtent(expressionX))
+      .range([0, width]);
+    const yScale = d3
+      .scaleLinear()
+      .domain(finiteExtent(expressionY))
+      .range([height, 0]);
+
+    return {
+      xScale,
+      yScale
+    };
+  }
+
+  reglDraw(regl, drawPoints, sizeBuffer, colorBuffer, pointBuffer, camera) {
+    regl.clear({
+      depth: 1,
+      color: [1, 1, 1, 1]
+    });
+
+    drawPoints({
+      size: sizeBuffer,
+      distance: camera.distance,
+      color: colorBuffer,
+      position: pointBuffer,
+      count: this.count,
+      view: camera.view()
+    });
+  }
+
+  drawAxesSVG(xScale, yScale, svg) {
+    const { scatterplotYYaccessor, scatterplotXXaccessor } = this.props;
+    svg.selectAll("*").remove();
+
+    // the axes are much cleaner and easier now. No need to rotate and orient
+    // the axis, just call axisBottom, axisLeft etc.
+    const xAxis = d3.axisBottom().scale(xScale);
+
+    const yAxis = d3.axisLeft().scale(yScale);
+
+    // adding axes is also simpler now, just translate x-axis to (0,height)
+    // and it's alread defined to be a bottom axis.
+    svg
+      .append("g")
+      .attr("transform", `translate(0,${height})`)
+      .attr("class", "x axis")
+      .call(xAxis);
+
+    // y-axis is translated to (0,0)
+    svg
+      .append("g")
+      .attr("transform", "translate(0,0)")
+      .attr("class", "y axis")
+      .call(yAxis);
+
+    // adding label. For x-axis, it's at (10, 10), and for y-axis at (width, height-10).
+    svg
+      .append("text")
+      .attr("x", 10)
+      .attr("y", 10)
+      .attr("class", "label")
+      .style("font-style", "italic")
+      .text(scatterplotYYaccessor);
+
+    svg
+      .append("text")
+      .attr("x", width)
+      .attr("y", height - 10)
+      .attr("text-anchor", "end")
+      .attr("class", "label")
+      .style("font-style", "italic")
+      .text(scatterplotXXaccessor);
+  }
+
+  render() {
+    const { dispatch } = this.props;
+    const { minimized } = this.state;
+
+    return (
+      <div
+        style={{
+          position: "fixed",
+          bottom: minimized ? -height + -margin.top : 0,
+          borderRadius: "3px 3px 0px 0px",
+          left: globals.leftSidebarWidth + globals.scatterplotMarginLeft,
+          padding: "0px 20px 20px 0px",
+          backgroundColor: "white",
+          /* x y blur spread color */
+          boxShadow: "0px 0px 6px 2px rgba(153,153,153,0.4)"
+        }}
+        id="scatterplot_wrapper"
+      >
+        <ButtonGroup
+          style={{
+            position: "absolute",
+            right: 5,
+            top: 5
+          }}
+        >
+          <Button
+            type="button"
+            minimal
+            onClick={() => {
+              this.setState({ minimized: !minimized });
+            }}
+          >
+            {minimized ? "show scatterplot" : "hide"}
+          </Button>
+          <Button
+            type="button"
+            minimal
+            onClick={() => {
+              dispatch({
+                type: "clear scatterplot"
+              });
+              dispatch({
+                type: "reset colorscale"
+              });
+            }}
+          >
+            remove
+          </Button>
+        </ButtonGroup>
+        <div
+          className={styles.scatterplot}
+          id="scatterplot"
+          style={{
+            width: `${width + margin.left + margin.right}px`,
+            height: `${height + margin.top + margin.bottom}px`
+          }}
+        >
+          <canvas
+            width={width}
+            height={height}
+            style={{
+              marginLeft: margin.left - 7,
+              marginTop: margin.top
+            }}
+            ref={canvas => {
+              this.reglCanvas = canvas;
+            }}
+          />
+        </div>
+      </div>
+    );
+  }
+}
+
+export default Scatterplot;

client/src/components/scatterplot/setupScatterplot.js

@@ -5,15 +5,17 @@
 ******************************************
 ******************************************/
 
-const setupScatterplot = (width, height, margin) => {
+import * as d3 from "d3";
-  var container = d3.select("#scatterplot");
 
-  var svg = container
+const setupScatterplot = (width, height, margin) => {
+  const container = d3.select("#scatterplot");
+
+  const svg = container
     .append("svg")
     .attr("width", width + margin.left + margin.right)
     .attr("height", height + margin.top + margin.bottom)
     .append("g")
-    .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
+    .attr("transform", `translate(${margin.left},${margin.top})`);
 
   return {
     svg

client/src/components/scatterplot/util.js

@@ -1,10 +1,5 @@
 // jshint esversion: 6
-import _ from "lodash";
+export const margin = { top: 40, right: 5, bottom: 20, left: 60 };
-
-const paddingRight = 120;
-const continuousChartWidth = 340;
-
-export const margin = { top: 66, right: 110, bottom: 20, left: 60 };
 export const width = 340;
 export const height = 340 - margin.top - margin.bottom;
 export const innerHeight = height - 2;

client/src/globals.js

@@ -1,3 +1,5 @@
+import { Colors } from "@blueprintjs/core";
+
 // jshint esversion: 6
 /* these will be either (preferably) specified or inferred */
 export const categories = [
@@ -28,9 +30,25 @@ export const continuous = [
   "Unmapped_short"
 ];
 
+/* if a categorical metadata field has more options than this, truncate */
+export const maxCategoricalOptionsToDisplay = 100;
+
+/*
+these are default values for configuration  the CLI may supply.
+See the REST API and CLI specs for more info.
+*/
+export const configDefaults = {
+  features: {},
+  displayNames: {},
+  parameters: {
+    "max-category-items": 1000
+  }
+};
+
 /* colors */
-export const blue = "#4a90e2";
+export const blue = Colors.BLUE3;
-export const hcaBlue = "#1c7cc7";
+export const linkBlue = Colors.BLUE5;
+export const lightestGrey = "rgb(249,249,249)";
 export const lighterGrey = "rgb(245,245,245)";
 export const lightGrey = "rgb(211,211,211)";
 export const mediumGrey = "rgb(153,153,153)";
@@ -41,36 +59,49 @@ export const brightBlue = "#4a90e2";
 export const brightGreen = "#A2D729";
 export const darkGreen = "#448C4D";
 
+export const nonFiniteCellColor = lightGrey;
+export const defaultCellColor = "rgb(0,0,0,1)";
+
+/* typography constants */
+
 export const tiniestFontSize = 12;
-
+export const largestFontSize = 24;
+export const uppercaseLetterSpacing = "0.04em";
 export const bolder = 700;
-
-export let API = {
-  // prefix: "http://api.clustering.czi.technology/api/",
-  //prefix: "http://tabulamuris.cxg.czi.technology/api/",
-   prefix: "http://pbmc3k.cxg.czi.technology/api/",
-  // prefix: "http://pbmc33k.cxg.czi.technology/api/",
-
-  // prefix: "http://api-staging.clustering.czi.technology/api/",
-  version: "v0.1/"
-};
-
-if (window.CELLXGENE && window.CELLXGENE.API) API = window.CELLXGENE.API;
-
-export let datasetTitle = "";
-
-if (window.CELLXGENE && window.CELLXGENE.datasetTitle)
-  datasetTitle = window.CELLXGENE.datasetTitle;
-
 export const accentFont = "Georgia,Times,Times New Roman,serif";
 export const maxParagraphWidth = 600;
+
+/* layout styles constants */
+
+export const cellxgeneTitleLeftPadding = 14;
+export const cellxgeneTitleTopPadding = 7;
+
 export const maxControlsWidth = 800;
 
 export const graphMargin = { top: 20, right: 10, bottom: 30, left: 40 };
-// export const graphWidth = 1440 /* window width */ - 410 /* sidebar */ - (15 + 15) /* left right padding */ /* but responsive */;
-// export const graphHeight = 500;
 export const graphWidth = 700;
 export const graphHeight = 700;
+export const scatterplotMarginLeft = 25;
+
+export const leftSidebarWidth = 365;
+export const leftSidebarSectionHeading = {
+  fontSize: 18,
+  textTransform: "uppercase",
+  fontWeight: 500,
+  letterSpacing: ".05em"
+};
+export const leftSidebarSectionPadding = 10;
+
+let _API = {
+  // prefix: "http://api.clustering.czi.technology/api/",
+  // prefix: "http://tabulamuris.cxg.czi.technology/api/",
+  // prefix: "http://api-staging.clustering.czi.technology/api/",
+  prefix: "http://localhost:5005/api/",
+  version: "v0.2/"
+};
+
+if (window.CELLXGENE && window.CELLXGENE.API) _API = window.CELLXGENE.API;
+export const API = _API;
 
 export const ordinalColors = [
   "#0ac115",

client/src/index.css

@@ -0,0 +1,9 @@
+/*
+>>>GLOBAL<<< CSS imports & definitions
+
+Do NOT put anything in this file that is not global.  We do not perform
+namespace mangling on these definitions.  See webpack config for specifics.
+*/
+@import "~@blueprintjs/core/lib/css/blueprint.css";
+@import "~@blueprintjs/icons/lib/css/blueprint-icons.css";
+@import "~@blueprintjs/select/lib/css/blueprint-select.css";

client/src/index.js

@@ -0,0 +1,21 @@
+// jshint esversion: 6
+/* eslint-disable no-console */
+import React from "react";
+import ReactDOM from "react-dom";
+import { Provider } from "react-redux";
+import { FocusStyleManager } from "@blueprintjs/core";
+
+import "./index.css";
+
+/* our code */
+import App from "./components/app";
+import store from "./reducers";
+
+FocusStyleManager.onlyShowFocusOnTabs();
+
+ReactDOM.render(
+  <Provider store={store}>
+    <App />
+  </Provider>,
+  document.getElementById("root")
+);

client/src/middleware/updateCellColors.js

@@ -0,0 +1,150 @@
+// jshint esversion: 6
+import _ from "lodash";
+import * as d3 from "d3";
+import { interpolateRainbow, interpolateCool } from "d3-scale-chromatic";
+import * as globals from "../globals";
+import parseRGB from "../util/parseRGB";
+import finiteExtent from "../util/finiteExtent";
+
+/*
+  https://medium.com/@jacobp100/you-arent-using-redux-middleware-enough-94ffe991e6
+  storeInstance =>
+    functionToCallWithAnActionThatWillSendItToTheNextMiddleware =>
+    actionThatDispatchWasCalledWith =>
+    valueToUseAsTheReturnValueOfTheDispatchCall
+*/
+
+/*
+  What this file does:
+
+  1. fire a filter action anywhere in the app
+  2. ** this middleware checks to see the state of all the currently selected filters,
+     including the new one
+  3. ** create updated selection from a copy of all the cells presently on the client
+     (this may be a subset of 'all')
+  4. ** append that new selection to the action so that it magically appears in the reducer
+     just because the action was fired
+
+  This is nice because we keep a lot of filtering business logic centralized
+  (what it means in practice to be selected)
+*/
+
+const updateCellColorsMiddleware = store => next => action => {
+  const s = store.getState();
+
+  /*
+  this is a hardcoded map of the things we need to keep an eye on and update
+  global cell selection in response to
+  */
+  const filterJustChanged =
+    action.type === "color by expression" ||
+    action.type === "color by continuous metadata" ||
+    action.type === "color by categorical metadata";
+
+  const obsAnnotations = _.get(s.controls, "world.obsAnnotations", null);
+  if (!filterJustChanged || !obsAnnotations) {
+    return next(
+      action
+    ); /* if the cells haven't loaded or the action wasn't a color change, bail */
+  }
+
+  let colorScale;
+  const colorsByRGB = new Array(obsAnnotations.length);
+
+  /*
+  in plain language...
+   (a) once the cells have loaded.
+   (b) each time a user changes a color control we need to update cellsMetadata colors
+  This is available to all the draw functions as controls.colorRGB[index]
+  */
+
+  if (action.type === "color by categorical metadata") {
+    const { categories } = _.filter(s.controls.world.schema.annotations.obs, {
+      name: action.colorAccessor
+    })[0];
+
+    colorScale = d3
+      .scaleSequential(interpolateRainbow)
+      .domain([0, categories.length]);
+
+    /* pre-create colors - much faster than doing it for each obs */
+    const colors = _.transform(categories, (acc, cat, idx) => {
+      acc[cat] = parseRGB(colorScale(idx));
+    });
+
+    const key = action.colorAccessor;
+    const col = obsAnnotations.col(key).asArray();
+    for (let i = 0, len = obsAnnotations.length; i < len; i += 1) {
+      const cat = col[i];
+      colorsByRGB[i] = colors[cat];
+    }
+  }
+
+  if (action.type === "color by continuous metadata") {
+    const colorBins = 100;
+    const { min, max } = action.rangeForColorAccessor;
+    colorScale = d3
+      .scaleQuantile()
+      .domain([min, max])
+      .range(_.range(colorBins - 1, -1, -1));
+
+    /* pre-create colors - much faster than doing it for each obs */
+    const colors = new Array(colorBins);
+    for (let i = 0; i < colorBins; i += 1) {
+      colors[i] = parseRGB(interpolateCool(i / colorBins));
+    }
+
+    const key = action.colorAccessor;
+    const nonFiniteColor = parseRGB(globals.nonFiniteCellColor);
+    const col = obsAnnotations.col(key).asArray();
+    for (let i = 0, len = obsAnnotations.length; i < len; i += 1) {
+      const val = col[i];
+      if (Number.isFinite(val)) {
+        const c = colorScale(val);
+        colorsByRGB[i] = colors[c];
+      } else {
+        colorsByRGB[i] = nonFiniteColor;
+      }
+    }
+  }
+
+  if (action.type === "color by expression") {
+    const { gene, data } = action;
+    const expression = data[gene]; // Float32Array
+    const colorBins = 100;
+    const [min, max] = finiteExtent(expression);
+    colorScale = d3
+      .scaleQuantile()
+      .domain([min, max])
+      .range(_.range(colorBins - 1, -1, -1));
+
+    /* pre-create colors - much faster than doing it for each obs */
+    const colors = new Array(colorBins);
+    for (let i = 0; i < colorBins; i += 1) {
+      colors[i] = parseRGB(interpolateCool(i / colorBins));
+    }
+    const nonFiniteColor = parseRGB(globals.nonFiniteCellColor);
+
+    for (let i = 0, len = expression.length; i < len; i += 1) {
+      const e = expression[i];
+      if (Number.isFinite(e)) {
+        const c = colorScale(e);
+        colorsByRGB[i] = colors[c];
+      } else {
+        colorsByRGB[i] = nonFiniteColor;
+      }
+    }
+  }
+
+  /*
+  append the result of all the filters to the action the user just triggered
+  */
+  const modifiedAction = Object.assign({}, action, {
+    colors: { rgb: colorsByRGB },
+    colorScale
+  });
+
+  return next(modifiedAction);
+};
+
+export default updateCellColorsMiddleware;

client/src/middleware/updateURLMiddleware.js

@@ -0,0 +1,87 @@
+// jshint esversion: 6
+// import uri from "urijs";
+
+/*
+NOTE: file currently not used, but retained as we expect to reinstate features in this
+area shortly.
+*/
+
+/*
+  https://medium.com/@jacobp100/you-arent-using-redux-middleware-enough-94ffe991e6
+  storeInstance
+    => functionToCallWithAnActionThatWillSendItToTheNextMiddleware
+    => actionThatDispatchWasCalledWith
+    => valueToUseAsTheReturnValueOfTheDispatchCall
+*/
+
+const updateURLMiddleware = (/* store */) => next => action => {
+  // const oldState = store.getState();
+  const nextAction = next(action);
+
+  if (action.type === "url changed") {
+    /* we don't handle pop state here - we handle it in the url reducer */
+    return nextAction;
+  }
+
+  // const state = store.getState();
+
+  /************************************************************************
+      *************************************************************************
+      1. Redux app state just changed. Clear URL, and then update it.
+      1a. We get the whole state tree to construct the url!
+      1b. But (see reducers/url.js) we try to centralize it because...
+      1c. ...the back button / initial load case ('url changed' return above)
+      means that we have to listen for 'url changed' and construct state
+      from the browser
+      *************************************************************************
+      ************************************************************************/
+
+  // const oldURI = URI(window.location.href)
+  // const newURI = URI(oldURI).setQuery({})
+
+  // if (window.location.search === "") {
+  //   newURL = uri.addQuery(category, value).toString(); /* #1 */
+  // } else if (uri.hasQuery(category, value) || uri.hasQuery(category, value, true)) { /* true param here means check arrays as well http://medialize.github.io/URI.js/docs.html#search-has */
+  //   newURL = uri.removeQuery(category, value).toString(); /* #4 */
+  // } else {
+  //   newURL = uri.addQuery(category, value).toString(); /* #2 & #3 are handled by URI */
+  // }
+  //
+  // window.history.pushState("", "", newURL)
+
+  //
+  // // Internal helper for working with URIs
+  // const oldURI = new URI(window.location.href);
+  // const newURI = new URI(oldURI).setQueryData({});
+  //
+  // newURI.setPath('/foo/bar');
+  //
+  // // Set the path based on state
+  // if (!state.isOnLandingPage && state.project.id) {
+  //   newURI.setPath(newURI.getPath() + state.project.id + '/');
+  //   newURI.addQueryData('baz', state.mode);
+  //   newURI.addQueryData('bat', state.selection.activePageID);
+  // } else {
+  //   newURI.setPath(newURI.getPath() + state.landingSection + '/');
+  // }
+  //
+  // // Avoid URL thrashing by replacing state while loading instead of pushing
+  // const newPath = newURI.toString();
+  // const oldPath = oldURI.toString();
+  // if (newPath !== oldPath) {
+  //   if (
+  //     (oldState.mode === 'asdf' &&
+  //       state.mode === 'asdf' &&
+  //       !oldState.isOnLandingPage) ||
+  //     oldState.isLoadingProject !== state.isLoadingProject
+  //   ) {
+  //     window.history.replaceState(null, null, newPath);
+  //   } else {
+  //     window.history.pushState(null, null, newPath);
+  //   }
+  // }
+
+  return nextAction;
+};
+
+export default updateURLMiddleware;

client/src/reducers/config.js

@@ -0,0 +1,34 @@
+// jshint esversion: 6
+const Config = (
+  state = {
+    displayNames: null,
+    features: null,
+    parameters: null
+  },
+  action
+) => {
+  switch (action.type) {
+    case "initial data load start":
+      return {
+        ...state,
+        loading: true,
+        error: null
+      };
+    case "configuration load complete":
+      return {
+        ...state,
+        loading: false,
+        error: null,
+        ...action.config
+      };
+    case "initial data load error":
+      return {
+        ...state,
+        error: action.error
+      };
+    default:
+      return state;
+  }
+};
+
+export default Config;

client/src/reducers/controls.js

@@ -0,0 +1,596 @@
+// jshint esversion: 6
+
+import _ from "lodash";
+
+import { World, WorldUtil, ControlsHelper } from "../util/stateManager";
+import parseRGB from "../util/parseRGB";
+import Crossfilter from "../util/typedCrossfilter";
+import * as globals from "../globals";
+import {
+  layoutDimensionName,
+  obsAnnoDimensionName,
+  userDefinedDimensionName,
+  diffexpDimensionName,
+  makeContinuousDimensionName
+} from "../util/nameCreators";
+
+const Controls = (
+  state = {
+    // data loading flag
+    loading: false,
+    error: null,
+
+    // configuration
+    maxCategoryItems: globals.configDefaults.parameters["max-category-items"],
+
+    // the whole big bang
+    universe: null,
+    fullUniverseCache: null,
+
+    // all of the data + selection state
+    world: null,
+    colorRGB: null,
+    categoricalSelectionState: null,
+    crossfilter: null,
+    dimensionMap: null,
+    userDefinedGenes: [],
+    userDefinedGenesLoading: false,
+    diffexpGenes: [],
+
+    colorAccessor: null,
+    colorScale: null,
+    resettingInterface: false,
+
+    opacityForDeselectedCells: 0.2,
+    graphBrushSelection: null,
+    continuousSelection: null,
+    scatterplotXXaccessor: null, // just easier to read
+    scatterplotYYaccessor: null,
+    axesHaveBeenDrawn: false,
+    graphRenderCounter: 0 /* integer as <Component key={graphRenderCounter} - a change in key forces a remount */,
+    __storedStateForCelllist1__: null /* will need procedural control of brush ie., brush.extent https://bl.ocks.org/micahstubbs/3cda05ca68cba260cb81 */,
+    __storedStateForCelllist2__: null
+  },
+  action
+) => {
+  /*
+  For now, log anything looking like an error to the console.
+  */
+  if (action.error || /error/i.test(action.type)) {
+    console.error(action.error);
+  }
+
+  switch (action.type) {
+    /*****************************************************
+          Initialization, World/Universe management
+          and data loading.
+    ******************************************************/
+    case "configuration load complete": {
+      // there are a couple of configuration items we need to retain
+      return {
+        ...state,
+        maxCategoryItems: _.get(
+          state.config,
+          "parameters.max-category-items",
+          globals.configDefaults.parameters["max-category-items"]
+        )
+      };
+    }
+    case "initial data load start": {
+      return { ...state, loading: true };
+    }
+    case "initial data load complete (universe exists)": {
+      /* first light - create world & other data-driven defaults */
+      const { universe } = action;
+      const world = World.createWorldFromEntireUniverse(universe);
+      const colorRGB = new Array(universe.nObs).fill(
+        parseRGB(globals.defaultCellColor)
+      );
+      const categoricalSelectionState = ControlsHelper.createCategoricalSelectionState(
+        state,
+        world
+      );
+      const crossfilter = Crossfilter(world.obsAnnotations);
+      const dimensionMap = World.createObsDimensionMap(crossfilter, world);
+      WorldUtil.clearCaches();
+
+      return {
+        ...state,
+        loading: false,
+        error: null,
+        universe,
+        fullUniverseCache: { world, crossfilter, dimensionMap },
+        world,
+        colorRGB,
+        categoricalSelectionState,
+        crossfilter,
+        dimensionMap,
+        colorAccessor: null,
+        resettingInterface: false
+      };
+    }
+    case "reset World to eq Universe": {
+      const {
+        userDefinedGenes,
+        diffexpGenes,
+        universe,
+        fullUniverseCache
+      } = state;
+      const { world, crossfilter } = fullUniverseCache;
+      // reset all crossfilter dimensions
+      _.forEach(fullUniverseCache.dimensionMap, dim => dim.filterAll());
+      const colorRGB = new Array(universe.nObs).fill(
+        parseRGB(globals.defaultCellColor)
+      );
+      const categoricalSelectionState = ControlsHelper.createCategoricalSelectionState(
+        state,
+        world
+      );
+
+      /* free dimensions not in cache (otherwise they leak) */
+      _.forEach(state.dimensionMap, (dim, dimName) => {
+        if (!fullUniverseCache.dimensionMap[dimName]) {
+          dim.dispose();
+        }
+      });
+      const dimensionMap = {
+        ...fullUniverseCache.dimensionMap,
+        ...ControlsHelper.createGenesDimMap(
+          userDefinedGenes,
+          diffexpGenes,
+          world,
+          crossfilter
+        )
+      };
+      WorldUtil.clearCaches();
+
+      return {
+        ...state,
+        world,
+        colorRGB,
+        categoricalSelectionState,
+        crossfilter,
+        dimensionMap,
+        colorAccessor: null,
+        resettingInterface: false
+      };
+    }
+    case "set World to current selection": {
+      const { userDefinedGenes, diffexpGenes } = state;
+
+      /* Set viewable world to be the currently selected data */
+      const world = World.createWorldFromCurrentSelection(
+        action.universe,
+        action.world,
+        action.crossfilter
+      );
+      const colorRGB = new Array(world.nObs).fill(
+        parseRGB(globals.defaultCellColor)
+      );
+      const categoricalSelectionState = ControlsHelper.createCategoricalSelectionState(
+        state,
+        world
+      );
+      const crossfilter = Crossfilter(world.obsAnnotations);
+      const dimensionMap = {
+        ...World.createObsDimensionMap(crossfilter, world),
+        ...ControlsHelper.createGenesDimMap(
+          userDefinedGenes,
+          diffexpGenes,
+          world,
+          crossfilter
+        )
+      };
+      WorldUtil.clearCaches();
+
+      return {
+        ...state,
+        loading: false,
+        error: null,
+        world,
+        colorRGB,
+        categoricalSelectionState,
+        crossfilter,
+        dimensionMap,
+        colorAccessor: null
+      };
+    }
+    case "expression load success": {
+      const { world, universe } = state;
+      let universeVarData = universe.varData;
+      let worldVarData = world.varData;
+
+      // Load new expression data into the varData dataframes, if
+      // not already present.
+      _.forEach(action.expressionData, (val, key) => {
+        // If not already in universe.varData, save entire expression column
+        if (!universeVarData.hasCol(key)) {
+          universeVarData = universeVarData.withCol(key, val);
+        }
+
+        // If not already in world.varData, save sliced expression column
+        if (!worldVarData.hasCol(key)) {
+          // Slice if world !== universe, else just use whole column.
+          // Use the obsAnnotation index as the cut key, as we keep
+          // all world dataframes in sync.
+          let worldValSlice = val;
+          if (!World.worldEqUniverse(world, universe)) {
+            worldValSlice = universeVarData
+              .subset(world.obsAnnotations.rowIndex.keys(), [key], null)
+              .icol(0)
+              .asArray();
+          }
+
+          // Now build world's varData dataframe
+          worldVarData = worldVarData.withCol(
+            key,
+            worldValSlice,
+            world.obsAnnotations.rowIndex
+          );
+        }
+      });
+
+      // Prune size of varData "cache" if getting out of hand....
+      const { userDefinedGenes, diffexpGenes } = state;
+      const allTheGenesWeNeed = _.uniq(
+        [].concat(
+          userDefinedGenes,
+          diffexpGenes,
+          Object.keys(action.expressionData)
+        )
+      );
+      universeVarData = ControlsHelper.pruneVarDataCache(
+        universeVarData,
+        allTheGenesWeNeed
+      );
+      worldVarData = ControlsHelper.pruneVarDataCache(
+        worldVarData,
+        allTheGenesWeNeed
+      );
+
+      return {
+        ...state,
+        universe: {
+          ...universe,
+          varData: universeVarData
+        },
+        world: {
+          ...world,
+          varData: worldVarData
+        }
+      };
+    }
+    case "request user defined gene started": {
+      return {
+        ...state,
+        userDefinedGenesLoading: true
+      };
+    }
+    case "request user defined gene error": {
+      return {
+        ...state,
+        userDefinedGenesLoading: false
+      };
+    }
+    case "request user defined gene success": {
+      const { world, crossfilter, dimensionMap, userDefinedGenes } = state;
+      const _userDefinedGenes = userDefinedGenes.slice();
+      const gene = action.data.genes[0];
+
+      dimensionMap[
+        userDefinedDimensionName(gene)
+      ] = World.createVarDataDimension(world, crossfilter, gene);
+
+      return {
+        ...state,
+        dimensionMap,
+        userDefinedGenes: _userDefinedGenes,
+        userDefinedGenesLoading: false
+      };
+    }
+    case "request differential expression success": {
+      const { world, crossfilter, dimensionMap } = state;
+      const _diffexpGenes = [];
+
+      action.data.forEach(d => {
+        _diffexpGenes.push(world.varAnnotations.at(d[0], "name"));
+      });
+
+      _.forEach(_diffexpGenes, gene => {
+        dimensionMap[diffexpDimensionName(gene)] = World.createVarDataDimension(
+          world,
+          crossfilter,
+          gene
+        );
+      });
+
+      return {
+        ...state,
+        dimensionMap,
+        diffexpGenes: _diffexpGenes
+      };
+    }
+    case "clear differential expression": {
+      const { world, dimensionMap } = state;
+      const _dimensionMap = dimensionMap;
+      _.forEach(action.diffExp, values => {
+        const name = world.varAnnotations.at(values[0], "name");
+        // clean up crossfilter dimensions
+        const dimension = dimensionMap[diffexpDimensionName(name)];
+        dimension.dispose();
+        delete dimensionMap[diffexpDimensionName(name)];
+      });
+      return {
+        ...state,
+        dimensionMap: _dimensionMap,
+        diffexpGenes: []
+      };
+    }
+    case "user defined gene": {
+      /*
+        this could also live in expression success with a conditional,
+        but that handles diffexp also
+      */
+      const newUserDefinedGenes = state.userDefinedGenes.slice();
+      newUserDefinedGenes.push(action.data);
+      return {
+        ...state,
+        userDefinedGenes: newUserDefinedGenes
+      };
+    }
+    case "clear user defined gene": {
+      const { userDefinedGenes, dimensionMap } = state;
+      const newUserDefinedGenes = _.filter(
+        userDefinedGenes,
+        d => d !== action.data
+      );
+
+      const dimension = dimensionMap[userDefinedDimensionName(action.data)];
+      dimension.dispose();
+      delete dimensionMap[userDefinedDimensionName(action.data)];
+
+      return {
+        ...state,
+        dimensionMap,
+        userDefinedGenes: newUserDefinedGenes
+      };
+    }
+    case "clear all user defined genes": {
+      const { userDefinedGenes, dimensionMap } = state;
+
+      _.forEach(userDefinedGenes, gene => {
+        const dimension = dimensionMap[userDefinedDimensionName(gene)];
+        dimension.dispose();
+        delete dimensionMap[userDefinedDimensionName(gene)];
+      });
+
+      return {
+        ...state,
+        dimensionMap,
+        userDefinedGenes: []
+      };
+    }
+    case "reset colorscale": {
+      const { world } = state;
+      const colorRGB = new Array(world.nObs).fill(
+        parseRGB(globals.defaultCellColor)
+      );
+      return {
+        ...state,
+        colorRGB,
+        colorAccessor: null
+      };
+    }
+    case "expression load error":
+    case "initial data load error": {
+      return {
+        ...state,
+        loading: false,
+        error: action.error
+      };
+    }
+
+    /*******************************
+             User Events
+     *******************************/
+    case "graph brush selection change": {
+      state.dimensionMap[layoutDimensionName("XY")].filterWithinRect(
+        action.brushCoords.northwest,
+        action.brushCoords.southeast
+      );
+      return {
+        ...state,
+        graphBrushSelection: action.brushCoords
+      };
+    }
+    case "lasso deselect":
+    case "graph brush deselect": {
+      state.dimensionMap[layoutDimensionName("XY")].filterAll();
+      return {
+        ...state,
+        graphBrushSelection: null
+      };
+    }
+    case "lasso selection": {
+      const { polygon } = action;
+      const dXY = state.dimensionMap[layoutDimensionName("XY")];
+      if (polygon.length < 3) {
+        // single point or a line is not a polygon, and is therefore a deselect
+        dXY.filterAll();
+      } else {
+        dXY.filterWithinPolygon(polygon);
+      }
+      return {
+        ...state
+      };
+    }
+    case "continuous metadata histogram brush": {
+      const name = makeContinuousDimensionName(
+        action.continuousNamespace,
+        action.selection
+      );
+
+      // action.selection: metadata name being selected
+      // action.range: filter range, or null if deselected
+      if (!action.range) {
+        state.dimensionMap[name].filterAll();
+      } else {
+        state.dimensionMap[name].filterRange(action.range);
+      }
+      return { ...state };
+    }
+    case "change opacity deselected cells in 2d graph background":
+      return {
+        ...state,
+        opacityForDeselectedCells: action.data
+      };
+    case "increment graph render counter": {
+      const c = state.graphRenderCounter + 1;
+      return {
+        ...state,
+        graphRenderCounter: c
+      };
+    }
+    case "interface reset started": {
+      return {
+        ...state,
+        resettingInterface: true
+      };
+    }
+    /*******************************
+          Categorical metadata
+    *******************************/
+    case "categorical metadata filter select": {
+      const newCategorySelected = Array.from(
+        state.categoricalSelectionState[action.metadataField].categorySelected
+      );
+      newCategorySelected[action.categoryIndex] = true;
+      const newCategoricalSelectionState = {
+        ...state.categoricalSelectionState,
+        [action.metadataField]: {
+          ...state.categoricalSelectionState[action.metadataField],
+          categorySelected: newCategorySelected
+        }
+      };
+
+      // update the filter to match all selected options
+      const cat = newCategoricalSelectionState[action.metadataField];
+      state.dimensionMap[obsAnnoDimensionName(action.metadataField)].filterEnum(
+        ControlsHelper.selectedValuesForCategory(cat)
+      );
+
+      return {
+        ...state,
+        categoricalSelectionState: newCategoricalSelectionState
+      };
+    }
+    case "categorical metadata filter deselect": {
+      const newCategorySelected = Array.from(
+        state.categoricalSelectionState[action.metadataField].categorySelected
+      );
+      newCategorySelected[action.categoryIndex] = false;
+      const newCategoricalSelectionState = {
+        ...state.categoricalSelectionState,
+        [action.metadataField]: {
+          ...state.categoricalSelectionState[action.metadataField],
+          categorySelected: newCategorySelected
+        }
+      };
+
+      // update the filter to match all selected options
+      const cat = newCategoricalSelectionState[action.metadataField];
+      state.dimensionMap[obsAnnoDimensionName(action.metadataField)].filterEnum(
+        ControlsHelper.selectedValuesForCategory(cat)
+      );
+
+      return {
+        ...state,
+        categoricalSelectionState: newCategoricalSelectionState
+      };
+    }
+    case "categorical metadata filter none of these": {
+      const newCategoricalSelectionState = {
+        ...state.categoricalSelectionState,
+        [action.metadataField]: {
+          ...state.categoricalSelectionState[action.metadataField],
+          categorySelected: Array.from(
+            state.categoricalSelectionState[action.metadataField]
+              .categorySelected
+          ).fill(false)
+        }
+      };
+      state.dimensionMap[
+        obsAnnoDimensionName(action.metadataField)
+      ].filterNone();
+      return {
+        ...state,
+        categoricalSelectionState: newCategoricalSelectionState
+      };
+    }
+    case "categorical metadata filter all of these": {
+      const newCategoricalSelectionState = {
+        ...state.categoricalSelectionState,
+        [action.metadataField]: {
+          ...state.categoricalSelectionState[action.metadataField],
+          categorySelected: Array.from(
+            state.categoricalSelectionState[action.metadataField]
+              .categorySelected
+          ).fill(true)
+        }
+      };
+      state.dimensionMap[
+        obsAnnoDimensionName(action.metadataField)
+      ].filterAll();
+      return {
+        ...state,
+        categoricalSelectionState: newCategoricalSelectionState
+      };
+    }
+
+    /*******************************
+              Color Scale
+    *******************************/
+    case "color by categorical metadata":
+    case "color by continuous metadata": {
+      return {
+        ...state,
+        colorRGB: action.colors.rgb,
+        colorAccessor: action.colorAccessor,
+        colorScale: action.colorScale
+      };
+    }
+    case "color by expression": {
+      return {
+        ...state,
+        colorRGB: action.colors.rgb,
+        colorAccessor: action.gene,
+        colorScale: action.colorScale
+      };
+    }
+
+    /*******************************
+              Scatterplot
+    *******************************/
+    case "set scatterplot x":
+      return {
+        ...state,
+        scatterplotXXaccessor: action.data
+      };
+    case "set scatterplot y":
+      return {
+        ...state,
+        scatterplotYYaccessor: action.data
+      };
+    case "clear scatterplot":
+      return {
+        ...state,
+        scatterplotXXaccessor: null,
+        scatterplotYYaccessor: null
+      };
+
+    default:
+      return state;
+  }
+};
+
+export default Controls;

client/src/reducers/differential.js

@@ -1,4 +1,5 @@
 // jshint esversion: 6
+
 const Differential = (
   state = {
     diffExp: null,
@@ -11,29 +12,49 @@ const Differential = (
 ) => {
   switch (action.type) {
     case "request differential expression started":
-      return Object.assign({}, state, {
+      return {
+        ...state,
         loading: true,
         error: null
-      });
+      };
     case "request differential expression success":
-      return Object.assign({}, state, {
+      return {
+        ...state,
         error: null,
         loading: false,
         diffExp: action.data
-      });
+      };
     case "request differential expression error":
-      return Object.assign({}, state, {
+      return {
+        ...state,
         loading: false,
         error: action.data
-      });
+      };
     case "store current cell selection as differential set 1":
-      return Object.assign({}, state, {
+      return {
+        ...state,
         celllist1: action.data
-      });
+      };
     case "store current cell selection as differential set 2":
-      return Object.assign({}, state, {
+      return {
+        ...state,
         celllist2: action.data
-      });
+      };
+    case "clear differential expression":
+      return {
+        ...state,
+        diffExp: null,
+        celllist1: null,
+        celllist2: null
+      };
+    case "reset World to eq Universe":
+    case "set World to current selection":
+      return {
+        ...state,
+        diffExp: null,
+        celllist1: null,
+        celllist2: null
+      };
     default:
       return state;
   }

client/src/reducers/index.js

@@ -0,0 +1,27 @@
+// jshint esversion: 6
+import { combineReducers, createStore, applyMiddleware } from "redux";
+import thunk from "redux-thunk";
+import { composeWithDevTools } from "redux-devtools-extension";
+import updateURLMiddleware from "../middleware/updateURLMiddleware";
+import updateCellColors from "../middleware/updateCellColors";
+
+import config from "./config";
+import differential from "./differential";
+import responsive from "./responsive";
+import controls from "./controls";
+
+const Reducer = combineReducers({
+  config,
+  responsive,
+  controls,
+  differential
+});
+
+const store = createStore(
+  Reducer,
+  composeWithDevTools(
+    applyMiddleware(thunk, updateURLMiddleware, updateCellColors)
+  )
+);
+
+export default store;

client/src/reducers/responsive.js

@@ -8,10 +8,11 @@ const Responsive = (
 ) => {
   switch (action.type) {
     case "window resize":
-      return Object.assign({}, state, {
+      return {
+        ...state,
         width: action.data.width,
         height: action.data.height
-      });
+      };
     default:
       return state;
   }

client/src/util/actionHelpers.js

@@ -0,0 +1,119 @@
+import _ from "lodash";
+/* XXX: cough, cough, ... */
+import { postNetworkErrorToast } from "../components/framework/toasters";
+
+/*
+dispatch an action error to the user.   Currently we use
+async toasts.
+*/
+export const dispatchNetworkErrorMessageToUser = message =>
+  postNetworkErrorToast(message);
+
+/*
+Catch unexpected errors and make sure we don't lose them!
+*/
+export function catchErrorsWrap(fn, dispatchToUser = false) {
+  return (dispatch, getState) => {
+    fn(dispatch, getState).catch(error => {
+      console.error(error);
+      if (dispatchToUser) {
+        dispatchNetworkErrorMessageToUser(error.message);
+      }
+      dispatch({ type: "UNEXPECTED ERROR", error });
+    });
+  };
+}
+
+/*
+Wrapper to perform async fetch with some modest error handling
+and decoding.
+*/
+const doFetch = async (url, acceptType) => {
+  const res = await fetch(url, {
+    method: "get",
+    headers: new Headers({
+      Accept: acceptType
+    })
+  });
+  if (res.ok && res.headers.get("Content-Type").includes(acceptType)) {
+    return res;
+  }
+  // else an error
+  let msg = `Unexpected HTTP response ${res.status}, ${res.statusText}`;
+  const body = await res.text();
+  if (body && body.length > 0) {
+    msg = `${msg} -- ${body}`;
+  }
+  dispatchNetworkErrorMessageToUser(msg);
+  throw new Error(msg);
+};
+
+/*
+Wrapper to perform an async fetch and JSON decode response.
+*/
+export const doJsonRequest = async url => {
+  const res = await doFetch(url, "application/json");
+  return res.json();
+};
+
+/*
+Wrapper to perform an async fetch for binary data.
+*/
+export const doBinaryRequest = async url => {
+  const res = await doFetch(url, "application/octet-stream");
+  return res.arrayBuffer();
+};
+
+/*
+This function "packs" filter index lists into the more efficient
+"range" form specified in the REST 0.2 spec.
+
+Specifically, it turns an array of indices [0, 1, 2, 10, 11, 14, ...]
+into a form that encodes runs of consecutive numbers as [min, max].
+Array may not be sorted, but will only contain uniq values.
+
+Parameters:
+   indices - input array of numbers (index)
+   minRangeLength - hint, min range length before it is encoded into range format.
+   sorted - boolean hint indicating array is presorted, ascending order
+
+So [1, 2, 3, 4, 10, 11, 14] -> [ [1, 4], [10, 11], 14]
+*/
+export const rangeEncodeIndices = (
+  indices,
+  minRangeLength = 3,
+  sorted = false
+) => {
+  if (indices.length === 0) {
+    return indices;
+  }
+
+  if (!sorted) {
+    indices = _.sortBy(indices);
+  }
+
+  const result = new Array(indices.length);
+  let resultTail = 0;
+
+  let i = 0;
+  while (i < indices.length) {
+    const begin = indices[i];
+    let current;
+    do {
+      current = indices[i];
+      i += 1;
+    } while (i < indices.length && indices[i] === current + 1);
+
+    if (current - begin + 1 >= minRangeLength) {
+      result[resultTail] = [begin, current];
+      resultTail += 1;
+    } else {
+      for (let j = begin; j <= current; j += 1, resultTail += 1) {
+        result[resultTail] = j;
+      }
+    }
+  }
+
+  result.length = resultTail;
+  return result;
+};

client/src/util/camera.js

@@ -1,9 +1,9 @@
 // jshint esversion: 6
-var createCamera = require("orbit-camera");
+const createCamera = require("orbit-camera");
-var createScroll = require("scroll-speed");
+const createScroll = require("scroll-speed");
-var mp = require("mouse-position");
+const mp = require("mouse-position");
-var mb = require("mouse-pressed");
+const mb = require("mouse-pressed");
-var key = require("key-pressed");
+const key = require("key-pressed");
 
 const panSpeed = 0.4;
 const scaleSpeed = 0.5;
@@ -17,20 +17,19 @@ function attachCamera(canvas, opts) {
   opts.scale = opts.scale !== false;
   opts.rotate = opts.rotate !== false;
 
-  var scroll = createScroll(canvas, opts.scale);
+  const scroll = createScroll(canvas, opts.scale);
-  var mbut = mb(canvas, opts.rotate);
+  const mbut = mb(canvas, opts.rotate);
-  var mpos = mp(canvas);
+  const mpos = mp(canvas);
-  var camera = createCamera([0, 0, 1], [0, 0, -1], [0, 1, 0]);
+  const camera = createCamera([0, 0, 1], [0, 0, -1], [0, 1, 0]);
 
   camera.tick = tick;
 
   return camera;
 
   function tick() {
-    var ctrl = key("<control>") || key("<alt>");
+    const ctrl = key("<control>") || key("<alt>");
-    var alt = key("<shift>");
+    const alt = key("<shift>");
-    var height = canvas.height;
+    const { height, width } = canvas;
-    var width = canvas.width;
 
     if (opts.rotate && mbut.left && ctrl && !alt) {
       camera.rotate(
@@ -41,23 +40,17 @@ function attachCamera(canvas, opts) {
 
     if ((opts.pan && mbut.right) || (mbut.left && !ctrl && !alt)) {
       camera.pan([
-        panSpeed *
+        ((panSpeed * (mpos[0] - mpos.prev[0])) / width) * camera.distance,
-          (mpos[0] - mpos.prev[0]) /
+        ((panSpeed * (mpos[1] - mpos.prev[1])) / height) * camera.distance
-          width *
-          Math.pow(camera.distance, 1),
-        panSpeed *
-          (mpos[1] - mpos.prev[1]) /
-          height *
-          Math.pow(camera.distance, 1)
       ]);
     }
 
     if (opts.scale && scroll[1]) {
-      camera.distance *= Math.exp(scroll[1] * scaleSpeed / height);
+      camera.distance *= Math.exp((scroll[1] * scaleSpeed) / height);
     }
 
     if (opts.scale && (mbut.middle || (mbut.left && !ctrl && alt))) {
-      var d = mpos.y - mpos.prevY;
+      const d = mpos.y - mpos.prevY;
       if (!d) return;
 
       camera.distance *= Math.exp(d / height);

client/src/util/dataframe/dataframe.js

@@ -0,0 +1,574 @@
+import { IdentityInt32Index, isLabelIndex } from "./labelIndex";
+// weird cross-dependency that we should clean up someday...
+import { sort } from "../typedCrossfilter/sort";
+import { isTypedArray, isArrayOrTypedArray, callOnceLazy } from "./util";
+import { summarizeContinuous, summarizeCategorical } from "./summarize";
+
+/*
+Dataframe is an immutable 2D matrix similiar to Python Pandas Dataframe,
+but (currently) without all of the surrounding support functions.
+Data is stored in column-major layout, and each column is monomorphic.
+
+It supports:
+* Relatively efficient creation, cloning and subsetting
+* Very efficient columnar access (eg, sum down a column), and access
+  to the underlying column arrays.
+* Data access by row/col offset or label.  Labels are reasonably well
+  optimized for both numeric lables and arbitrary (eg, sting) labels.
+
+It does not currently support:
+* Views on matrix subset - for currently known access patterns,
+  it is more effiicent to copy on subsetting, optimizing for access
+  speed over memory use.
+* JS iterators - they are too slow.  Use explicit iteration over
+  offest or labels.
+
+Important assumptions embedded in the API:
+* Columns are implicitly categorical if they are a JS Array and numeric
+  (aka continuous) if they are a TypedArray.
+
+There are three index types for row/col indexing:
+* IdentityInt32Index - noop index, where the index label is the offset.
+* KeyIndex - index arbitrary JS objects.
+* DenseInt32Index - integer indexing.  Optimization over KeyIndex as it uses
+  Int32Array as a back-map to offsets.  This means that the index array
+  must be sized to [minLabel, maxLabel), so this is only useful when the label
+  range is relatively close the underlying offset range [minOffset, maxOffset).
+
+All private functions/methods/fields are prefixed by '__', eg, __compile().
+Don't use them outside of this file.
+
+Simple example:
+
+  // default indexing is integer offset.
+  const df = Dataframe.create([2,2], [['a', 'b'], [0, 1]])
+  console.log(df.at(0,0));  // outputs: a
+  console.log(df.col(1).asArray());   // outputs: [0, 1]
+
+  // KeyIndex
+  const df = new Dataframe([1,2], [['a'], ['b']], null, new KeyIndex(['A', 'B']))
+  console.log(df.at(0, 'A')); // outputs: a
+  console.log(df.col('A').asArray(); // outputs: ['a']
+
+Performance tuning is primarily focused on columnar access patterns, which is the
+dominant pattern in cellxgene.
+*/
+
+/**
+Dataframe
+**/
+
+class Dataframe {
+  /**
+  Constructors & factories
+  **/
+
+  constructor(dims, columnarData, rowIndex = null, colIndex = null) {
+    /*
+    The base constructor is relatively hard to use - as an alternative,
+    see factory methods and clone/slice, below.
+
+    Parameters:
+      * dims - 2D array describing intendend dimensionality: [nRows,nCols].
+      * columnarData - JS array, nCols in length, containing array
+        or TypedArray of length nRows.
+      * rowIndex/colIndex - null (create default index using offsets as key),
+        or a caller-provided index.
+    All columns and indices must have appropriate dimensionality.
+    */
+    const [nRows, nCols] = dims;
+    if (nRows < 0 || nCols < 0) {
+      throw new RangeError("Dataframe dimensions must be positive");
+    }
+    if (!rowIndex) {
+      rowIndex = new IdentityInt32Index(nRows);
+    }
+    if (!colIndex) {
+      colIndex = new IdentityInt32Index(nCols);
+    }
+    Dataframe.__errorChecks(dims, columnarData, rowIndex, colIndex);
+
+    this.__columns = Array.from(columnarData);
+    this.dims = dims;
+    this.length = nRows; // convenience accessor for row dimension
+    this.rowIndex = rowIndex;
+    this.colIndex = colIndex;
+
+    this.__compile();
+  }
+
+  static __errorChecks(dims, columnarData, rowIndex, colIndex) {
+    const [nRows, nCols] = dims;
+
+    /* check for expected types */
+    if (!Array.isArray(columnarData)) {
+      throw new TypeError("Dataframe constructor requires array of columns");
+    }
+    if (!columnarData.every(c => isArrayOrTypedArray(c))) {
+      throw new TypeError("Dataframe columns must all be Array or TypedArray");
+    }
+    if (!isLabelIndex(rowIndex)) {
+      throw new TypeError("Dataframe rowIndex is an unsupported type.");
+    }
+    if (!isLabelIndex(colIndex)) {
+      throw new TypeError("Dataframe colIndex is an unsupported type.");
+    }
+
+    /* check for expected dimensionality / size */
+    if (
+      nCols !== columnarData.length ||
+      !columnarData.every(c => c.length === nRows)
+    ) {
+      throw new RangeError(
+        "Dataframe dimension does not match provided data shape"
+      );
+    }
+    if (nRows !== rowIndex.size()) {
+      throw new RangeError(
+        "Dataframe rowIndex must have same size as underlying data"
+      );
+    }
+    if (nCols !== colIndex.size()) {
+      throw new RangeError(
+        "Dataframe colIndex must have same size as underlying data"
+      );
+    }
+  }
+
+  __compile() {
+    /*
+    Compile data accessors for each column.
+
+    Each column accessor is a function which will lookup data by
+    index (ie, is equivalent to dataframe.get(row, col), where 'col'
+    is fixed.
+
+    In addition, each column accessor has several functions:
+
+    asArray() -- return the entire column as a native Array or TypedArray.
+      Crucially, this native array only supports label indexing.
+      Example:
+        const arr = df.col('a').asArray();
+
+    has(rlabel) -- return boolean indicating of the row label
+      is contained within the column.  Example:
+        const isInColumn = df.col('a').includes(99)
+      For the default offset indexing, this is identical to:
+        const isInColumn = (99 > 0) && (99 < df.nRows);
+
+    ihas(roffset) -- same as has(), but accepts a row offset
+      instead of a row label.
+
+    indexOf(value) -- return the label (not offset) of the first instance of
+      'value' in the column.  If you want the offset, just use the builtin JS
+      indexOf() function, available on both Array and TypedArray.
+
+    iget(offset) -- return the value at 'offset'
+
+    */
+    const { getOffset, getLabel } = this.rowIndex;
+    this.__columnsAccessor = this.__columns.map(column => {
+      const { length } = column;
+
+      /* get value by row label */
+      const get = function get(rlabel) {
+        return column[getOffset(rlabel)];
+      };
+
+      /* get value by row offset */
+      const iget = function iget(roffset) {
+        return column[roffset];
+      };
+
+      /* full column array access */
+      const asArray = function asArray() {
+        return column;
+      };
+
+      /* test for row label inclusion in column */
+      const has = function has(rlabel) {
+        const offset = getOffset(rlabel);
+        return offset >= 0 && offset < length;
+      };
+
+      const ihas = function ihas(offset) {
+        return offset >= 0 && offset < length;
+      };
+
+      /*
+      return first label (index) at which the value is found in this column,
+      or undefined if not found.
+
+      NOTE: not found return is DIFFERENT than the default Array.indexOf as
+      -1 is a plausible Dataframe row/col label.
+      */
+      const indexOf = function indexOf(value) {
+        const offset = column.indexOf(value);
+        if (offset === -1) {
+          return undefined;
+        }
+        return getLabel(offset);
+      };
+
+      /*
+      Summarize the column data. Lazy eval;
+      */
+      const summarize = callOnceLazy(() =>
+        isTypedArray(column)
+          ? summarizeContinuous(column)
+          : summarizeCategorical(column)
+      );
+
+      get.summarize = summarize;
+      get.asArray = asArray;
+      get.has = has;
+      get.ihas = ihas;
+      get.indexOf = indexOf;
+      get.iget = iget;
+      return get;
+    });
+  }
+
+  clone() {
+    /*
+    Clone this dataframe
+    */
+    return new this.constructor(
+      this.dims,
+      [...this.__columns],
+      this.rowIndex,
+      this.colIndex
+    );
+  }
+
+  withCol(label, colData, withRowIndex = null) {
+    /*
+    Create a new DF, which is `this` plus the new column. Example:
+    const newDf = df.withCol("foo", [1,2,3]);
+
+    Dimensionality of new column must match existing dataframe.
+
+    Special case: empty dataframe will accept any size column.  Example:
+    const newDf = Dataframe.empty().withCol("foo", [1,2,3]);
+
+    If `withRowIndex` specified, the provided index will become the
+    rowIndex for the newly created dataframe.   If not specified,
+    the rowIndex from `this` will be used (ie, the rowIndex is
+    unchanged).
+    */
+    let dims;
+    let rowIndex;
+    if (this.isEmpty()) {
+      dims = [colData.length, 1];
+      rowIndex = null;
+    } else {
+      dims = [this.dims[0], this.dims[1] + 1];
+      ({ rowIndex } = this);
+    }
+
+    if (withRowIndex) {
+      rowIndex = withRowIndex;
+    }
+
+    const columns = [...this.__columns];
+    columns.push(colData);
+    const colIndex = this.colIndex.withLabel(label);
+    return new this.constructor(dims, columns, rowIndex, colIndex);
+  }
+
+  dropCol(label) {
+    /*
+    Create a new dataframe, omitting one columns.
+
+    const newDf = df.dropCol("colors");
+    */
+    const dims = [this.dims[0], this.dims[1] - 1];
+    const coffset = this.colIndex.getOffset(label);
+    const columns = [...this.__columns];
+    columns.splice(coffset, 1);
+    const colIndex = this.colIndex.dropLabel(label);
+    return new this.constructor(dims, columns, this.rowIndex, colIndex);
+  }
+
+  static empty(rowIndex = null, colIndex = null) {
+    return new Dataframe([0, 0], [], rowIndex, colIndex);
+  }
+
+  static create(dims, columnarData) {
+    /*
+    Create a dataframe from raw columnar data.  All column arrays
+    must have the same length.   Identity indexing will be used.
+
+    Example:
+    const df = Dataframe.create([2,2], [new Uint32Array(2), new Float32Array(2)]);
+    */
+    return new Dataframe(dims, columnarData, null, null);
+  }
+
+  __subset(rowOffsets, colOffsets, withRowIndex) {
+    const dims = [...this.dims];
+
+    const getSortedLabelAndOffsets = (offsets, index) => {
+      /*
+      Given offsets, return both offsets and associated lables,
+      sorted by offset.
+      */
+      if (!offsets) {
+        return [null, null];
+      }
+      const sortedOffsets = sort(offsets);
+      const sortedLabels = new Array(sortedOffsets.length);
+      for (let i = 0, l = sortedOffsets.length; i < l; i += 1) {
+        sortedLabels[i] = index.getLabel(sortedOffsets[i]);
+      }
+      return [sortedLabels, sortedOffsets];
+    };
+
+    let { colIndex } = this;
+    if (colOffsets) {
+      let colLabels;
+      [colLabels, colOffsets] = getSortedLabelAndOffsets(
+        colOffsets,
+        this.colIndex
+      );
+      dims[1] = colOffsets.length;
+      colIndex = this.colIndex.subsetLabels(colLabels);
+    }
+
+    let { rowIndex } = this;
+    if (withRowIndex) rowIndex = withRowIndex;
+    if (rowOffsets) {
+      let rowLabels;
+      [rowLabels, rowOffsets] = getSortedLabelAndOffsets(
+        rowOffsets,
+        this.rowIndex
+      );
+      dims[0] = rowLabels.length;
+      if (!withRowIndex) rowIndex = this.rowIndex.subsetLabels(rowLabels);
+    }
+
+    /* subset columns */
+    let columns = this.__columns;
+    if (colOffsets) {
+      columns = new Array(colOffsets.length);
+      for (let i = 0, l = colOffsets.length; i < l; i += 1) {
+        columns[i] = this.__columns[colOffsets[i]];
+      }
+    }
+
+    /* subset rows */
+    if (rowOffsets) {
+      columns = columns.map(col => {
+        const newCol = new col.constructor(rowOffsets.length);
+        for (let i = 0, l = rowOffsets.length; i < l; i += 1) {
+          newCol[i] = col[rowOffsets[i]];
+        }
+        return newCol;
+      });
+    }
+    return new Dataframe(dims, columns, rowIndex, colIndex);
+  }
+
+  subset(rowLabels, colLabels = null, withRowIndex = null) {
+    /*
+    Subset by row/col labels.
+
+    withRowIndex allows assignment of new row index during subset operation.
+    If withRowIndex === null, it will reset the index to identity (offset)
+    indexing.  if withRowIndex is a label index object, it will be used
+    for the new dataframe.
+    */
+    const toOffsets = (labels, index) => {
+      if (!labels) {
+        return null;
+      }
+      return labels.map(label => {
+        const off = index.getOffset(label);
+        if (off === undefined) {
+          throw new RangeError(`unknown label: ${label}`);
+        }
+        return off;
+      });
+    };
+
+    const rowOffsets = toOffsets(rowLabels, this.rowIndex);
+    const colOffsets = toOffsets(colLabels, this.colIndex);
+    return this.__subset(rowOffsets, colOffsets, withRowIndex);
+  }
+
+  isubset(rowOffsets, colOffsets = null, withRowIndex = null) {
+    /*
+    Subset by row/col offset.
+
+    withRowIndex allows assignment of new row index during subset operation.
+    If withRowIndex === null, it will reset the index to identity (offset)
+    indexing.  if withRowIndex is a label index object, it will be used
+    for the new dataframe.
+    */
+    return this.__subset(rowOffsets, colOffsets, withRowIndex);
+  }
+
+  isubsetMask(rowMask, colMask = null, withRowIndex = null) {
+    /*
+    Subset on row/column based upon a truthy/falsey array (a mask).
+
+    withRowIndex allows assignment of new row index during subset operation.
+    If withRowIndex === null, it will reset the index to identity (offset)
+    indexing.  if withRowIndex is a label index object, it will be used
+    for the new dataframe.
+    */
+    const [nRows, nCols] = this.dims;
+    if (
+      (rowMask && rowMask.length !== nRows) ||
+      (colMask && colMask.length !== nCols)
+    ) {
+      throw new RangeError("boolean arrays must match row/col dimensions");
+    }
+
+    /* convert masks to lists - method wastes space, but is fast */
+    const toList = (mask, maxSize) => {
+      if (!mask) {
+        return null;
+      }
+      const list = new Int32Array(maxSize);
+      let elems = 0;
+      for (let i = 0, l = mask.length; i < l; i += 1) {
+        if (mask[i]) {
+          list[elems] = i;
+          elems += 1;
+        }
+      }
+      return new Int32Array(list.buffer, 0, elems);
+    };
+    const rowOffsets = toList(rowMask, nRows);
+    const colOffsets = toList(colMask, nCols);
+    return this.__subset(rowOffsets, colOffsets, withRowIndex);
+  }
+
+  /**
+  Data access with row/col.
+  **/
+
+  col(columnLabel) {
+    /*
+    Return accessor bound to a column.  Allows random row access
+    based upon the row indexing.  Returns undefined if the
+    columnLabel is not present in the dataframe.
+
+    Example for a dataframe with string labeled columns, and
+    default (offset) indices for rows (eg, [0, 'foo'])
+
+      const getValue = df.col('foo');
+      for (let r = 0; r < df.nRows; r += 1) {
+        console.log(r, getValue(r));
+      }
+
+    See __compile() for the functions available in a column accessor.
+    */
+    const coff = this.colIndex.getOffset(columnLabel);
+    return this.__columnsAccessor[coff];
+  }
+
+  icol(columnOffset) {
+    /*
+    Return column accessor by offset.
+    */
+    return this.__columnsAccessor[columnOffset];
+  }
+
+  at(r, c) {
+    /*
+    Access a single value, for a row/col label pair.
+
+    For performance reasons, there are no bounds or existance
+    checks on labels, and no defined behavior when these are supplied.
+    May return undefined, throw an Error, or do something else for
+    non-existant labels.  If you want predictable out-of-bounds
+    behavior, use has(), eg,
+
+    const myVal = df.has(r,l) ? df.at(r,l) : undefined;
+    */
+    const coff = this.colIndex.getOffset(c);
+    const roff = this.rowIndex.getOffset(r);
+    return this.__columns[coff][roff];
+  }
+
+  iat(r, c) {
+    /*
+    Access a single value, for a row/col offset (integer) position.
+
+    For performance reasons, there are no bounds checks on row/col offsets
+    or other well-defined behavior for out-of-bounds values.  If you want
+    well-defined bounds checking, use ihas(), eg,
+
+    const myVal = df.ihas(r, c) ? df.iat(r, c) : undefined;
+    */
+    return this.__columns[c][r];
+  }
+
+  has(r, c) {
+    /*
+    Test if row/col labels exist in the dataframe - returns true/false
+    */
+    const [nRows, nCols] = this.dims;
+    const coff = this.colIndex.getOffset(c);
+    const roff = this.rowIndex.getOffset(r);
+    return coff >= 0 && coff < nCols && roff >= 0 && roff < nRows;
+  }
+
+  ihas(r, c) {
+    /*
+    Test if row/col offset (integer) position exists in the
+    dataframe - returns true/false
+    */
+    const [nRows, nCols] = this.dims;
+    return c >= 0 && c < nCols && r >= 0 && r < nRows;
+  }
+
+  hasCol(c) {
+    /*
+    Test if col label exists - return true/false
+    */
+    return !!this.col(c);
+  }
+
+  isEmpty() {
+    /*
+    Return true if this is an empty dataframe, ie, has dimensions [0,0]
+    */
+    const [rows, cols] = this.dims;
+    return rows === 0 && cols === 0;
+  }
+
+  /****
+  Functional (map/reduce/etc) data access
+
+  XXX: not yet implemented, as there is no clear use case.   Can easily
+  add these as useful.
+  ****/
+
+  /*
+  Map & reduce of column or row
+
+  XXX TODO remainder of map/reduce functions:  mapCol, mapRow, reduceRow, ...
+  */
+  /* comment out until we have a use for this
+
+  reduceCol(clabel, callback, initialValue) {
+    const coff = this.colIndex.getOffset(clabel);
+    const column = this.__columns[coff];
+    let start = 0;
+    let acc = initialValue;
+    if (initialValue === undefined) {
+      acc = column[0];
+      start = 1;
+    }
+    for (let i = start, l = column.length; i < l; i += 1) {
+      acc = callback(acc, column[i]);
+    }
+    return acc;
+  }
+  */
+}
+
+export default Dataframe;

client/src/util/dataframe/index.js

@@ -0,0 +1,2 @@
+export { default as Dataframe } from "./dataframe";
+export { DenseInt32Index, IdentityInt32Index, KeyIndex } from "./labelIndex";

client/src/util/dataframe/labelIndex.js

@@ -0,0 +1,244 @@
+/**
+Label indexing - map a label to & from an integer offset.  See Dataframe
+for how this is used.
+**/
+
+/*
+Private utility functions
+*/
+function extent(tarr) {
+  let min = 0x7fffffff;
+  let max = ~min; // eslint-disable-line no-bitwise
+  for (let i = 0, l = tarr.length; i < l; i += 1) {
+    const v = tarr[i];
+    if (v < min) {
+      min = v;
+    }
+    if (v > max) {
+      max = v;
+    }
+  }
+  return [min, max];
+}
+
+function fillRange(arr, start = 0) {
+  const larr = arr;
+  for (let i = 0, l = larr.length; i < l; i += 1) {
+    larr[i] = i + start;
+  }
+  return larr;
+}
+
+/* eslint-disable class-methods-use-this */
+class IdentityInt32Index {
+  /*
+  identity/noop index, with small assumptions that labels are int32
+  */
+  constructor(maxOffset) {
+    this.maxOffset = maxOffset;
+  }
+
+  keys() {
+    // memoize
+    const k = fillRange(new Int32Array(this.maxOffset));
+    this.keys = function keys() {
+      return k;
+    };
+    return k;
+  }
+
+  getOffset(i) {
+    // label to offset
+    return i;
+  }
+
+  getLabel(i) {
+    // offset to label
+    return i;
+  }
+
+  size() {
+    return this.maxOffset;
+  }
+
+  __promote(labelArray) {
+    /*
+    time/space decision - based on the resulting density
+    */
+    const [minLabel, maxLabel] = extent(labelArray);
+    const labelSpaceSize = maxLabel - minLabel + 1;
+    const density = labelSpaceSize / this.maxOffset;
+    /* 0.1 is a magic number, that needs testing to optimize */
+    if (density < 0.1) {
+      return new KeyIndex(labelArray);
+    }
+    return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
+  }
+
+  subsetLabels(labelArray) {
+    return this.__promote(labelArray);
+  }
+
+  withLabel(label) {
+    if (label === this.maxOffset) {
+      return new IdentityInt32Index(label + 1);
+    }
+    return this.__promote([...this.keys(), label]);
+  }
+
+  dropLabel(label) {
+    if (label === this.maxOffset - 1) {
+      return new IdentityInt32Index(label);
+    }
+    const labelArray = [...this.keys()];
+    labelArray.splice(labelArray.indexOf(label), 1);
+    return this.__promote(labelArray);
+  }
+}
+/* eslint-enable class-methods-use-this */
+
+/* eslint-disable class-methods-use-this */
+class DenseInt32Index {
+  /*
+  DenseInt32Index indexes integer labels, and uses Int32Array typed arrays
+  for both forward and reverse indexing.   This means that the min/max range
+  of the forward index labels must be known a priori (so that the index
+  array can be pre-allocated).
+  */
+  constructor(labels, labelRange = null) {
+    if (labels.constructor !== Int32Array) {
+      labels = new Int32Array(labels);
+    }
+
+    if (!labelRange) {
+      labelRange = extent(labels);
+    }
+    const [minLabel, maxLabel] = labelRange;
+    const labelSpaceSize = maxLabel - minLabel + 1;
+    const index = new Int32Array(labelSpaceSize).fill(-1);
+    for (let i = 0, l = labels.length; i < l; i += 1) {
+      const label = labels[i];
+      index[label - minLabel] = i;
+    }
+
+    this.minLabel = minLabel;
+    this.rindex = labels;
+    this.index = index;
+    this.__compile();
+  }
+
+  __compile() {
+    const { minLabel, index, rindex } = this;
+    this.getOffset = function getOffset(l) {
+      return index[l - minLabel];
+    };
+    this.getLabel = function getLabel(i) {
+      return rindex[i];
+    };
+  }
+
+  keys() {
+    return this.rindex;
+  }
+
+  size() {
+    return this.rindex.length;
+  }
+
+  __promote(labelArray) {
+    /*
+    time/space decision - if we are going to use less than 10% of the
+    dense index space, switch to a KeyIndex (which is slower, but uses
+    less memory for sparse label spaces).
+    */
+    const [minLabel, maxLabel] = extent(labelArray);
+    const labelSpaceSize = maxLabel - minLabel + 1;
+    const density = labelSpaceSize / this.rindex.length;
+    /* 0.1 is a magic number, that needs testing to optimize */
+    if (density < 0.1) {
+      return new KeyIndex(labelArray);
+    }
+    return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
+  }
+
+  subsetLabels(labelArray) {
+    return this.__promote(labelArray);
+  }
+
+  withLabel(label) {
+    return this.__promote([...this.keys(), label]);
+  }
+
+  dropLabel(label) {
+    const labelArray = [...this.keys()];
+    labelArray.splice(labelArray.indexOf(label), 1);
+    return this.__promote(labelArray);
+  }
+}
+/* eslint-enable class-methods-use-this */
+
+/* eslint-disable class-methods-use-this */
+class KeyIndex {
+  /*
+  KeyIndex indexes arbitrary JS primitive types, and uses a Map()
+  as its core data structure.
+  */
+  constructor(labels) {
+    const index = new Map();
+    if (labels === undefined) {
+      labels = [];
+    }
+    const rindex = labels;
+    labels.forEach((v, i) => {
+      index.set(v, i);
+    });
+
+    this.index = index;
+    this.rindex = rindex;
+    this.__compile();
+  }
+
+  __compile() {
+    const { index, rindex } = this;
+    this.getOffset = function getOffset(k) {
+      return index.get(k);
+    };
+    this.getLabel = function getLabel(i) {
+      return rindex[i];
+    };
+  }
+
+  keys() {
+    return this.rindex;
+  }
+
+  size() {
+    return this.rindex.length;
+  }
+
+  subsetLabels(labelArray) {
+    return new KeyIndex(labelArray);
+  }
+
+  withLabel(label) {
+    return new KeyIndex([...this.rindex, label]);
+  }
+
+  dropLabel(label) {
+    const idx = this.rindex.indexOf(label);
+    const labelArray = [...this.rindex];
+    labelArray.splice(idx, 1);
+    return new KeyIndex(labelArray);
+  }
+}
+/* eslint-enable class-methods-use-this */
+
+function isLabelIndex(i) {
+  return (
+    i instanceof IdentityInt32Index ||
+    i instanceof DenseInt32Index ||
+    i instanceof KeyIndex
+  );
+}
+
+export { DenseInt32Index, IdentityInt32Index, KeyIndex, isLabelIndex };

client/src/util/dataframe/summarize.js

@@ -0,0 +1,57 @@
+/*
+Private dataframe support functions
+*/
+
+export function summarizeContinuous(col) {
+  let min;
+  let max;
+  let nan = 0;
+  let pinf = 0;
+  let ninf = 0;
+  if (col) {
+    for (let r = 0, l = col.length; r < l; r += 1) {
+      const val = Number(col[r]);
+      if (Number.isFinite(val)) {
+        if (min === undefined) {
+          min = val;
+          max = val;
+        } else {
+          min = val < min ? val : min;
+          max = val > max ? val : max;
+        }
+      } else if (Number.isNaN(val)) {
+        nan += 1;
+      } else if (val > 0) {
+        pinf += 1;
+      } else {
+        ninf += 1;
+      }
+    }
+  }
+  return {
+    categorical: false,
+    min,
+    max,
+    nan,
+    pinf,
+    ninf
+  };
+}
+
+export function summarizeCategorical(col) {
+  const categoryCounts = new Map();
+  if (col) {
+    for (let r = 0, l = col.length; r < l; r += 1) {
+      const val = col[r];
+      let curCount = categoryCounts.get(val);
+      if (curCount === undefined) curCount = 0;
+      categoryCounts.set(val, curCount + 1);
+    }
+  }
+  return {
+    categorical: true,
+    categories: [...categoryCounts.keys()],
+    categoryCounts,
+    numCategories: categoryCounts.size
+  };
+}

client/src/util/dataframe/util.js

@@ -0,0 +1,28 @@
+/*
+Private utility code for dataframe
+*/
+
+export function isTypedArray(x) {
+  return (
+    ArrayBuffer.isView(x) &&
+    Object.prototype.toString.call(x) !== "[object DataView]"
+  );
+}
+
+export function isArrayOrTypedArray(x) {
+  return Array.isArray(x) || isTypedArray(x);
+}
+
+export function callOnceLazy(f) {
+  let value;
+  let calledOnce = false;
+  const result = function result(...args) {
+    if (!calledOnce) {
+      value = f(...args);
+      calledOnce = true;
+    }
+    return value;
+  };
+
+  return result;
+}

client/src/util/finiteExtent.js

@@ -0,0 +1,33 @@
+/*
+Return the [minimum, maximum] extent, of the given typed array, ignoring
+non-finite values (ie, +Infinity, -Infinity).
+
+If undefined or empty array, or array contains only non-finite numbers,
+will return [undefined, undefined]
+*/
+
+function finiteExtent(tarr) {
+  let min;
+  let max;
+  let i;
+
+  for (i = 0; i < tarr.length; i += 1) {
+    const val = tarr[i];
+    if (Number.isFinite(val)) {
+      min = val;
+      max = val;
+      i += 1;
+      break;
+    }
+  }
+  for (; i < tarr.length; i += 1) {
+    const val = tarr[i];
+    if (Number.isFinite(val)) {
+      if (min > val) min = val;
+      if (max < val) max = val;
+    }
+  }
+  return [min, max];
+}
+
+export default finiteExtent;

client/src/util/nameCreators.js

@@ -0,0 +1,45 @@
+/*
+
+The original motivation for this file is name collision. It's possible
+to have two of the same gene, user defined and diffexpressed, on the client
+simultaneously, looking at the same data in the cache but with two
+dimensions and two crossfilters. It's also possible that someone could
+have a obsAnnotation named X, but we are using that for layout. So
+we namespace, and abstract to avoid proliferating strings throughout the
+codebase.
+
+*/
+
+const makeDimensionName = (namespace, key) => `${namespace}_${key}`;
+
+export const layoutDimensionName = key => makeDimensionName("layout", key);
+export const obsAnnoDimensionName = key => makeDimensionName("obsAnno", key);
+export const diffexpDimensionName = key =>
+  makeDimensionName("varData_diffexp", key);
+export const userDefinedDimensionName = key =>
+  makeDimensionName("varData_userDefined", key);
+
+/*
+    continuousNamespace = {
+    isObs: true,
+    isDiffExp: false,
+    isUserDefined: false
+  }
+
+  ie., makeContinuousDimensionName(continuousNamespace = {isObs: true}, "total_reads")
+  see: histogram brush, as it doesn't know what type of continuous it was with only field
+*/
+export const makeContinuousDimensionName = (continuousNamespace, key) => {
+  let name;
+  if (continuousNamespace.isObs) {
+    name = obsAnnoDimensionName(key);
+  } else if (continuousNamespace.isDiffExp) {
+    name = diffexpDimensionName(key);
+  } else if (continuousNamespace.isUserDefined) {
+    name = userDefinedDimensionName(key);
+  } else {
+    throw new Error("unknown continuous dimension");
+  }
+
+  return name;
+};

client/src/util/parseRGB.js

@@ -1,31 +1,30 @@
-// jshint esversion: 6
+// jshint esversion: 6
-import { scaleRGB } from "./scaleRGB";
+import scaleRGB from "./scaleRGB";
-
+
-// maintain a cache of already parsed RGB names, as it is reasonably expensive
+// maintain a cache of already parsed RGB names, as it is reasonably expensive
-// to do this operation.  This lets us have speed, but keep the pleasant ability
+// to do this operation.  This lets us have speed, but keep the pleasant ability
-// to talk about colors by their text description eg, 'rgb(0,0,1)'
+// to talk about colors by their text description eg, 'rgb(0,0,1)'
-//
+//
-const colorCache = new Object(null); // no prototype
+const colorCache = {};
-
+
-function parseColorName(c) {
+function parseColorName(c) {
-  if (c[0] !== "#") {
+  if (c[0] !== "#") {
-    const _c = c.replace(/[^\d,.]/g, "").split(",");
+    const _c = c.replace(/[^\d,.]/g, "").split(",");
-    return [scaleRGB(+_c[0]), scaleRGB(+_c[1]), scaleRGB(+_c[2])];
+    return [scaleRGB(+_c[0]), scaleRGB(+_c[1]), scaleRGB(+_c[2])];
-  } else {
+  }
-    var parsedHex = /^#?([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i.exec(c);
+  const parsedHex = /^#?([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i.exec(c);
-    return [
+  return [
-      scaleRGB(parseInt(parsedHex[1], 16)),
+    scaleRGB(parseInt(parsedHex[1], 16)),
-      scaleRGB(parseInt(parsedHex[2], 16)),
+    scaleRGB(parseInt(parsedHex[2], 16)),
-      scaleRGB(parseInt(parsedHex[3], 16))
+    scaleRGB(parseInt(parsedHex[3], 16))
-    ];
+  ];
-  }
+}
-}
+
-
+export default c => {
-export const parseRGB = c => {
+  let cv = colorCache[c];
-  var cv = colorCache[c];
+  if (!cv) {
-  if (!cv) {
+    cv = parseColorName(c);
-    cv = parseColorName(c);
+    colorCache[c] = cv;
-    colorCache[c] = cv;
+  }
-  }
+  return cv;
-  return cv;
+};
-};

client/src/util/scaleLinear.js

@@ -1,18 +1,16 @@
-// jshint esversion: 6
+// jshint esversion: 6
-
+
-// Substitute for a d3 linear scale - less flexible, more performant.
+// Substitute for a d3 linear scale - less flexible, more performant.
-// Returns a function which will scale a value.
+// Returns a function which will scale a value.
-//
+//
-// Example will scale [0,1] to [-1,1]
+// Example will scale [0,1] to [-1,1]
-//     var myScale = scaleLinear([0, 1], [-1, 1]);
+//     var myScale = scaleLinear([0, 1], [-1, 1]);
-//     myScale(0) === -1
+//     myScale(0) === -1
-// this is is equivalent to d3.scaleLinear().domain([0,1]).range([-1,1])
+// this is is equivalent to d3.scaleLinear().domain([0,1]).range([-1,1])
-
+
-export const scaleLinear = (domain, range) => {
+export default (domain, range) => {
-  const domainStart = domain[0];
+  const domainStart = domain[0];
-  const scale = (range[1] - range[0]) / (domain[1] - domain[0]);
+  const scale = (range[1] - range[0]) / (domain[1] - domain[0]);
-  const rangeStart = range[0];
+  const rangeStart = range[0];
-  return function(value) {
+  return value => (value - domainStart) * scale + rangeStart;
-    return (value - domainStart) * scale + rangeStart;
+};
-  };
-};

client/src/util/scaleRGB.js

@@ -1,5 +1,5 @@
 // jshint esversion: 6
-export const scaleRGB = input => {
+export default input => {
   const outputMax = 1;
   const outputMin = 0;
 

client/src/util/stateManager/controlsHelpers.js

@@ -0,0 +1,164 @@
+/*
+Helper functions for the controls reducer
+*/
+
+import _ from "lodash";
+
+import * as globals from "../../globals";
+import { fillRange } from "../typedCrossfilter/util";
+import {
+  userDefinedDimensionName,
+  diffexpDimensionName
+} from "../nameCreators";
+import * as World from "./world";
+
+/*
+Selection state for categoricals are tracked in an Object that
+has two main components for each category:
+1. mapping of option value to an index
+2. array of bool selection state by index
+Remember that option values can be ANY js type, except undefined/null.
+
+  {
+    _category_name_1: {
+      // map of option value to index
+      categoryIndices: Map([
+        catval1: index,
+        ...
+      ])
+
+      // index->selection true/false state
+      categorySelected: [ true/false, true/false, ... ]
+
+      // number of options
+      numCategories: number,
+
+      // isTruncated - true if the options for selection has
+      // been truncated (ie, was too large to implement)
+    }
+  }
+*/
+function topNCategories(summary) {
+  const counts = _.map(summary.categories, cat =>
+    summary.categoryCounts.get(cat)
+  );
+  const sortIndex = fillRange(new Array(summary.numCategories)).sort(
+    (a, b) => counts[b] - counts[a]
+  );
+  const sortedCategories = _.map(sortIndex, i => summary.categories[i]);
+  const sortedCounts = _.map(sortIndex, i => counts[i]);
+  const N = globals.maxCategoricalOptionsToDisplay;
+
+  if (sortedCategories.length < N) {
+    return [sortedCategories, sortedCounts];
+  }
+  return [sortedCategories.slice(0, N), sortedCounts.slice(0, N)];
+}
+
+export function createCategoricalSelectionState(state, world) {
+  const res = {};
+  _.forEach(world.obsAnnotations.colIndex.keys(), key => {
+    const summary = world.obsAnnotations.col(key).summarize();
+    if (summary.categories) {
+      const isColorField = key.includes("color") || key.includes("Color");
+      const isSelectableCategory =
+        !isColorField &&
+        key !== "name" &&
+        summary.categories.length < state.maxCategoryItems;
+      if (isSelectableCategory) {
+        const [categoryValues, categoryCounts] = topNCategories(summary);
+        const categoryIndices = new Map(categoryValues.map((v, i) => [v, i]));
+        const numCategories = categoryIndices.size;
+        const categorySelected = new Array(numCategories).fill(true);
+        const isTruncated = categoryValues.length < summary.numCategories;
+        res[key] = {
+          categoryValues, // array: of natively typed category values
+          categoryIndices, // map: category value (native type) -> category index
+          categorySelected, // array: t/f selection state
+          numCategories, // number: of categories
+          isTruncated, // bool: true if list was truncated
+          categoryCounts // array: cardinality of each category
+        };
+      }
+    }
+  });
+  return res;
+}
+
+/*
+given a categoricalSelectionState, return the list of all category values
+where selection state is true (ie, they are selected).
+*/
+export function selectedValuesForCategory(categorySelectionState) {
+  const selectedValues = _([...categorySelectionState.categoryIndices])
+    .filter(tuple => categorySelectionState.categorySelected[tuple[1]])
+    .map(tuple => tuple[0])
+    .value();
+  return selectedValues;
+}
+
+/*
+build a crossfilter dimension map for all gene expression related dimensions.
+*/
+export function createGenesDimMap(
+  userDefinedGenes,
+  diffexpGenes,
+  world,
+  crossfilter
+) {
+  function _createGenesDimMap(genes, nameCreator) {
+    return genes.reduce((acc, gene) => {
+      acc[nameCreator(gene)] = World.createVarDataDimension(
+        world,
+        crossfilter,
+        gene
+      );
+      return acc;
+    }, {});
+  }
+
+  return {
+    ..._createGenesDimMap(userDefinedGenes, userDefinedDimensionName),
+    ..._createGenesDimMap(diffexpGenes, diffexpDimensionName)
+  };
+}
+
+export function pruneVarDataCache(varData, needed) {
+  /*
+  Remove any unneeded columns from the varData dataframe.  Will only
+  prune / remove if the total column count exceeds VarDataCacheLowWatermark
+
+  Note: this code leverages the fact that dataframe offsets indicate
+  the order in which the columns were added.  This crudely provides
+  LRU semantics, so we can delete "older" columns first.
+  */
+
+  /*
+  VarDataCacheLowWatermark - this cofig value sets the minimum cache size,
+  in columns, below which we don't throw away data.
+
+  The value should be high enough so we are caching the maximum which will
+  "typically" be used in the UI (currently: 10 for diffexp, and N for user-
+  specified genes), and low enough to account for memory use (any single
+  column size is 4 bytes * numObs, so a column can be multi-megabyte in common
+  use cases).
+  */
+  const VarDataCacheLowWatermark = 32;
+
+  const numOverWatermark = varData.dims[1] - VarDataCacheLowWatermark;
+  if (numOverWatermark <= 0) return varData;
+
+  const { colIndex } = varData;
+  const all = colIndex.keys();
+  const unused = _.difference(all, needed);
+  if (unused.length > 0) {
+    // sort by offset in the dataframe - ie, psuedo-LRU
+    unused.sort((a, b) => colIndex.getOffset(a) - colIndex.getOffset(b));
+    const numToDrop =
+      unused.length < numOverWatermark ? unused.length : numOverWatermark;
+    for (let i = 0; i < numToDrop; i += 1) {
+      varData = varData.dropCol(unused[i]);
+    }
+  }
+  return varData;
+}

client/src/util/stateManager/index.js

@@ -0,0 +1,20 @@
+// jshint esversion: 6
+
+/*
+Model manager providing an abstraction for the use of the reducer code.
+This module provides several buckets of functionality:
+  - schema and config driven tranformation of the wire protocol
+    into a format that is easy for the UI code to use.
+  - manage the universe/world abstraction:
+    + universe: all of the server-provided, read-only data
+    + world: subset of universe
+  - lazy access and caching of dataframe contents as needed
+
+This is all VERY tightly integrated with reducers and actions, and
+exists to support those concepts.
+*/
+
+export * as Universe from "./universe";
+export * as World from "./world";
+export * as WorldUtil from "./worldUtil";
+export * as ControlsHelper from "./controlsHelpers";

client/src/util/stateManager/matrix.js

@@ -0,0 +1,75 @@
+import { flatbuffers } from "flatbuffers";
+import { NetEncoding } from "./matrix_generated";
+
+const utf8Decoder = new TextDecoder("utf-8");
+
+/*
+Matrix flatbuffer decoding support.   See fbs/matrix.fbs
+*/
+
+/*
+Decode NetEncoding.TypedArray
+*/
+function decodeTypedArray(uType, uValF, inplace = false) {
+  if (uType === NetEncoding.TypedArray.NONE) {
+    return null;
+  }
+
+  // Convert to a JS class that supports this type
+  const TypeClass = NetEncoding[NetEncoding.TypedArray[uType]];
+  // Create a TypedArray that references the underlying buffer
+  let arr = uValF(new TypeClass()).dataArray();
+  if (uType === NetEncoding.TypedArray.JSONEncodedArray) {
+    const json = utf8Decoder.decode(arr);
+    arr = JSON.parse(json);
+  } else if (!inplace) {
+    /* force copy to release underlying FBS buffer */
+    arr = new arr.constructor(arr);
+  }
+  return arr;
+}
+
+/*
+Parameter: Uint8Array or ArrayBuffer containing raw flatbuffer Matrix
+Returns: object containing decoded Matrix:
+{
+  nRows: num,
+  nCols: num,
+  columns: [
+    each column, which will be a TypedArray or Array
+  ]
+  colIdx: []|null
+}
+*/
+function decodeMatrixFBS(arrayBuffer, inplace = false) {
+  const bb = new flatbuffers.ByteBuffer(new Uint8Array(arrayBuffer));
+  const df = NetEncoding.Matrix.getRootAsMatrix(bb);
+
+  const nRows = df.nRows();
+  const nCols = df.nCols();
+
+  /* decode columns */
+  const columnsLength = df.columnsLength();
+  const columns = Array(columnsLength).fill(null);
+  for (let c = 0; c < columnsLength; c += 1) {
+    const col = df.columns(c);
+    columns[c] = decodeTypedArray(col.uType(), col.u.bind(col), inplace);
+  }
+
+  /* decode col_idx */
+  const colIdx = decodeTypedArray(
+    df.colIndexType(),
+    df.colIndex.bind(df),
+    inplace
+  );
+
+  return {
+    nRows,
+    nCols,
+    columns,
+    colIdx,
+    rowIdx: null
+  };
+}
+
+export default decodeMatrixFBS;

client/src/util/stateManager/matrix_generated.js

@@ -0,0 +1,835 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/**
+ * @const
+ * @namespace
+ */
+var NetEncoding = NetEncoding || {};
+
+/**
+ * @enum
+ */
+NetEncoding.TypedArray = {
+  NONE: 0, 0: 'NONE',
+  Float32Array: 1, 1: 'Float32Array',
+  Int32Array: 2, 2: 'Int32Array',
+  Uint32Array: 3, 3: 'Uint32Array',
+  Float64Array: 4, 4: 'Float64Array',
+  JSONEncodedArray: 5, 5: 'JSONEncodedArray'
+};
+
+/**
+ * @constructor
+ */
+NetEncoding.Float32Array = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {NetEncoding.Float32Array}
+ */
+NetEncoding.Float32Array.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {NetEncoding.Float32Array=} obj
+ * @returns {NetEncoding.Float32Array}
+ */
+NetEncoding.Float32Array.getRootAsFloat32Array = function(bb, obj) {
+  return (obj || new NetEncoding.Float32Array).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @param {number} index
+ * @returns {number}
+ */
+NetEncoding.Float32Array.prototype.data = function(index) {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.readFloat32(this.bb.__vector(this.bb_pos + offset) + index * 4) : 0;
+};
+
+/**
+ * @returns {number}
+ */
+NetEncoding.Float32Array.prototype.dataLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @returns {Float32Array}
+ */
+NetEncoding.Float32Array.prototype.dataArray = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? new Float32Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+NetEncoding.Float32Array.startFloat32Array = function(builder) {
+  builder.startObject(1);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} dataOffset
+ */
+NetEncoding.Float32Array.addData = function(builder, dataOffset) {
+  builder.addFieldOffset(0, dataOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {Array.<number>} data
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Float32Array.createDataVector = function(builder, data) {
+  builder.startVector(4, data.length, 4);
+  for (var i = data.length - 1; i >= 0; i--) {
+    builder.addFloat32(data[i]);
+  }
+  return builder.endVector();
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+NetEncoding.Float32Array.startDataVector = function(builder, numElems) {
+  builder.startVector(4, numElems, 4);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Float32Array.endFloat32Array = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @constructor
+ */
+NetEncoding.Uint32Array = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {NetEncoding.Uint32Array}
+ */
+NetEncoding.Uint32Array.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {NetEncoding.Uint32Array=} obj
+ * @returns {NetEncoding.Uint32Array}
+ */
+NetEncoding.Uint32Array.getRootAsUint32Array = function(bb, obj) {
+  return (obj || new NetEncoding.Uint32Array).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @param {number} index
+ * @returns {number}
+ */
+NetEncoding.Uint32Array.prototype.data = function(index) {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.readUint32(this.bb.__vector(this.bb_pos + offset) + index * 4) : 0;
+};
+
+/**
+ * @returns {number}
+ */
+NetEncoding.Uint32Array.prototype.dataLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @returns {Uint32Array}
+ */
+NetEncoding.Uint32Array.prototype.dataArray = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? new Uint32Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+NetEncoding.Uint32Array.startUint32Array = function(builder) {
+  builder.startObject(1);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} dataOffset
+ */
+NetEncoding.Uint32Array.addData = function(builder, dataOffset) {
+  builder.addFieldOffset(0, dataOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {Array.<number>} data
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Uint32Array.createDataVector = function(builder, data) {
+  builder.startVector(4, data.length, 4);
+  for (var i = data.length - 1; i >= 0; i--) {
+    builder.addInt32(data[i]);
+  }
+  return builder.endVector();
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+NetEncoding.Uint32Array.startDataVector = function(builder, numElems) {
+  builder.startVector(4, numElems, 4);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Uint32Array.endUint32Array = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @constructor
+ */
+NetEncoding.Int32Array = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {NetEncoding.Int32Array}
+ */
+NetEncoding.Int32Array.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {NetEncoding.Int32Array=} obj
+ * @returns {NetEncoding.Int32Array}
+ */
+NetEncoding.Int32Array.getRootAsInt32Array = function(bb, obj) {
+  return (obj || new NetEncoding.Int32Array).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @param {number} index
+ * @returns {number}
+ */
+NetEncoding.Int32Array.prototype.data = function(index) {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.readInt32(this.bb.__vector(this.bb_pos + offset) + index * 4) : 0;
+};
+
+/**
+ * @returns {number}
+ */
+NetEncoding.Int32Array.prototype.dataLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @returns {Int32Array}
+ */
+NetEncoding.Int32Array.prototype.dataArray = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? new Int32Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+NetEncoding.Int32Array.startInt32Array = function(builder) {
+  builder.startObject(1);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} dataOffset
+ */
+NetEncoding.Int32Array.addData = function(builder, dataOffset) {
+  builder.addFieldOffset(0, dataOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {Array.<number>} data
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Int32Array.createDataVector = function(builder, data) {
+  builder.startVector(4, data.length, 4);
+  for (var i = data.length - 1; i >= 0; i--) {
+    builder.addInt32(data[i]);
+  }
+  return builder.endVector();
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+NetEncoding.Int32Array.startDataVector = function(builder, numElems) {
+  builder.startVector(4, numElems, 4);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Int32Array.endInt32Array = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @constructor
+ */
+NetEncoding.Float64Array = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {NetEncoding.Float64Array}
+ */
+NetEncoding.Float64Array.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {NetEncoding.Float64Array=} obj
+ * @returns {NetEncoding.Float64Array}
+ */
+NetEncoding.Float64Array.getRootAsFloat64Array = function(bb, obj) {
+  return (obj || new NetEncoding.Float64Array).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @param {number} index
+ * @returns {number}
+ */
+NetEncoding.Float64Array.prototype.data = function(index) {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.readFloat64(this.bb.__vector(this.bb_pos + offset) + index * 8) : 0;
+};
+
+/**
+ * @returns {number}
+ */
+NetEncoding.Float64Array.prototype.dataLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @returns {Float64Array}
+ */
+NetEncoding.Float64Array.prototype.dataArray = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? new Float64Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+NetEncoding.Float64Array.startFloat64Array = function(builder) {
+  builder.startObject(1);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} dataOffset
+ */
+NetEncoding.Float64Array.addData = function(builder, dataOffset) {
+  builder.addFieldOffset(0, dataOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {Array.<number>} data
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Float64Array.createDataVector = function(builder, data) {
+  builder.startVector(8, data.length, 8);
+  for (var i = data.length - 1; i >= 0; i--) {
+    builder.addFloat64(data[i]);
+  }
+  return builder.endVector();
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+NetEncoding.Float64Array.startDataVector = function(builder, numElems) {
+  builder.startVector(8, numElems, 8);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Float64Array.endFloat64Array = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @constructor
+ */
+NetEncoding.JSONEncodedArray = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {NetEncoding.JSONEncodedArray}
+ */
+NetEncoding.JSONEncodedArray.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {NetEncoding.JSONEncodedArray=} obj
+ * @returns {NetEncoding.JSONEncodedArray}
+ */
+NetEncoding.JSONEncodedArray.getRootAsJSONEncodedArray = function(bb, obj) {
+  return (obj || new NetEncoding.JSONEncodedArray).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @param {number} index
+ * @returns {number}
+ */
+NetEncoding.JSONEncodedArray.prototype.data = function(index) {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.readUint8(this.bb.__vector(this.bb_pos + offset) + index) : 0;
+};
+
+/**
+ * @returns {number}
+ */
+NetEncoding.JSONEncodedArray.prototype.dataLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @returns {Uint8Array}
+ */
+NetEncoding.JSONEncodedArray.prototype.dataArray = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? new Uint8Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+NetEncoding.JSONEncodedArray.startJSONEncodedArray = function(builder) {
+  builder.startObject(1);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} dataOffset
+ */
+NetEncoding.JSONEncodedArray.addData = function(builder, dataOffset) {
+  builder.addFieldOffset(0, dataOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {Array.<number>} data
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.JSONEncodedArray.createDataVector = function(builder, data) {
+  builder.startVector(1, data.length, 1);
+  for (var i = data.length - 1; i >= 0; i--) {
+    builder.addInt8(data[i]);
+  }
+  return builder.endVector();
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+NetEncoding.JSONEncodedArray.startDataVector = function(builder, numElems) {
+  builder.startVector(1, numElems, 1);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.JSONEncodedArray.endJSONEncodedArray = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @constructor
+ */
+NetEncoding.Column = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {NetEncoding.Column}
+ */
+NetEncoding.Column.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {NetEncoding.Column=} obj
+ * @returns {NetEncoding.Column}
+ */
+NetEncoding.Column.getRootAsColumn = function(bb, obj) {
+  return (obj || new NetEncoding.Column).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @returns {NetEncoding.TypedArray}
+ */
+NetEncoding.Column.prototype.uType = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? /** @type {NetEncoding.TypedArray} */ (this.bb.readUint8(this.bb_pos + offset)) : NetEncoding.TypedArray.NONE;
+};
+
+/**
+ * @param {flatbuffers.Table} obj
+ * @returns {?flatbuffers.Table}
+ */
+NetEncoding.Column.prototype.u = function(obj) {
+  var offset = this.bb.__offset(this.bb_pos, 6);
+  return offset ? this.bb.__union(obj, this.bb_pos + offset) : null;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+NetEncoding.Column.startColumn = function(builder) {
+  builder.startObject(2);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {NetEncoding.TypedArray} uType
+ */
+NetEncoding.Column.addUType = function(builder, uType) {
+  builder.addFieldInt8(0, uType, NetEncoding.TypedArray.NONE);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} uOffset
+ */
+NetEncoding.Column.addU = function(builder, uOffset) {
+  builder.addFieldOffset(1, uOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Column.endColumn = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @constructor
+ */
+NetEncoding.Matrix = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {NetEncoding.Matrix}
+ */
+NetEncoding.Matrix.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {NetEncoding.Matrix=} obj
+ * @returns {NetEncoding.Matrix}
+ */
+NetEncoding.Matrix.getRootAsMatrix = function(bb, obj) {
+  return (obj || new NetEncoding.Matrix).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @returns {number}
+ */
+NetEncoding.Matrix.prototype.nRows = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.readUint32(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @returns {number}
+ */
+NetEncoding.Matrix.prototype.nCols = function() {
+  var offset = this.bb.__offset(this.bb_pos, 6);
+  return offset ? this.bb.readUint32(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @param {number} index
+ * @param {NetEncoding.Column=} obj
+ * @returns {NetEncoding.Column}
+ */
+NetEncoding.Matrix.prototype.columns = function(index, obj) {
+  var offset = this.bb.__offset(this.bb_pos, 8);
+  return offset ? (obj || new NetEncoding.Column).__init(this.bb.__indirect(this.bb.__vector(this.bb_pos + offset) + index * 4), this.bb) : null;
+};
+
+/**
+ * @returns {number}
+ */
+NetEncoding.Matrix.prototype.columnsLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 8);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @returns {NetEncoding.TypedArray}
+ */
+NetEncoding.Matrix.prototype.colIndexType = function() {
+  var offset = this.bb.__offset(this.bb_pos, 10);
+  return offset ? /** @type {NetEncoding.TypedArray} */ (this.bb.readUint8(this.bb_pos + offset)) : NetEncoding.TypedArray.NONE;
+};
+
+/**
+ * @param {flatbuffers.Table} obj
+ * @returns {?flatbuffers.Table}
+ */
+NetEncoding.Matrix.prototype.colIndex = function(obj) {
+  var offset = this.bb.__offset(this.bb_pos, 12);
+  return offset ? this.bb.__union(obj, this.bb_pos + offset) : null;
+};
+
+/**
+ * @returns {NetEncoding.TypedArray}
+ */
+NetEncoding.Matrix.prototype.rowIndexType = function() {
+  var offset = this.bb.__offset(this.bb_pos, 14);
+  return offset ? /** @type {NetEncoding.TypedArray} */ (this.bb.readUint8(this.bb_pos + offset)) : NetEncoding.TypedArray.NONE;
+};
+
+/**
+ * @param {flatbuffers.Table} obj
+ * @returns {?flatbuffers.Table}
+ */
+NetEncoding.Matrix.prototype.rowIndex = function(obj) {
+  var offset = this.bb.__offset(this.bb_pos, 16);
+  return offset ? this.bb.__union(obj, this.bb_pos + offset) : null;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+NetEncoding.Matrix.startMatrix = function(builder) {
+  builder.startObject(7);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} nRows
+ */
+NetEncoding.Matrix.addNRows = function(builder, nRows) {
+  builder.addFieldInt32(0, nRows, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} nCols
+ */
+NetEncoding.Matrix.addNCols = function(builder, nCols) {
+  builder.addFieldInt32(1, nCols, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} columnsOffset
+ */
+NetEncoding.Matrix.addColumns = function(builder, columnsOffset) {
+  builder.addFieldOffset(2, columnsOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {Array.<flatbuffers.Offset>} data
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Matrix.createColumnsVector = function(builder, data) {
+  builder.startVector(4, data.length, 4);
+  for (var i = data.length - 1; i >= 0; i--) {
+    builder.addOffset(data[i]);
+  }
+  return builder.endVector();
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+NetEncoding.Matrix.startColumnsVector = function(builder, numElems) {
+  builder.startVector(4, numElems, 4);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {NetEncoding.TypedArray} colIndexType
+ */
+NetEncoding.Matrix.addColIndexType = function(builder, colIndexType) {
+  builder.addFieldInt8(3, colIndexType, NetEncoding.TypedArray.NONE);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} colIndexOffset
+ */
+NetEncoding.Matrix.addColIndex = function(builder, colIndexOffset) {
+  builder.addFieldOffset(4, colIndexOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {NetEncoding.TypedArray} rowIndexType
+ */
+NetEncoding.Matrix.addRowIndexType = function(builder, rowIndexType) {
+  builder.addFieldInt8(5, rowIndexType, NetEncoding.TypedArray.NONE);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} rowIndexOffset
+ */
+NetEncoding.Matrix.addRowIndex = function(builder, rowIndexOffset) {
+  builder.addFieldOffset(6, rowIndexOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Matrix.endMatrix = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} offset
+ */
+NetEncoding.Matrix.finishMatrixBuffer = function(builder, offset) {
+  builder.finish(offset);
+};
+
+// Exports for ECMAScript6 Modules
+export {NetEncoding};

client/src/util/stateManager/universe.js

@@ -0,0 +1,148 @@
+// jshint esversion: 6
+
+import _ from "lodash";
+
+import decodeMatrixFBS from "./matrix";
+import * as Dataframe from "../dataframe";
+
+/*
+Private helper function - create and return a template Universe
+*/
+function templateUniverse() {
+  /* default universe template */
+  return {
+    nObs: 0,
+    nVar: 0,
+    schema: {},
+
+    /*
+    Annotations
+    */
+    obsAnnotations: Dataframe.Dataframe.empty(),
+    varAnnotations: Dataframe.Dataframe.empty(),
+    obsLayout: Dataframe.Dataframe.empty(),
+
+    /*
+    Var data columns - subset of all
+    */
+    varData: Dataframe.Dataframe.empty(null, new Dataframe.KeyIndex())
+  };
+}
+
+/*
+This module implements functions that support storage of "Universe",
+aka all of the var/obs data and annotations.
+
+These functions are used exclusively by the actions and reducers to
+build an internal POJO for use by the rendering components.
+*/
+
+function AnnotationsFBSToDataframe(arrayBuffer) {
+  /*
+  Convert a Matrix FBS to a Dataframe.
+  */
+  const fbs = decodeMatrixFBS(arrayBuffer);
+  const df = new Dataframe.Dataframe(
+    [fbs.nRows, fbs.nCols],
+    fbs.columns,
+    null,
+    new Dataframe.KeyIndex(fbs.colIdx)
+  );
+  return df;
+}
+
+function LayoutFBSToDataframe(arrayBuffer) {
+  const fbs = decodeMatrixFBS(arrayBuffer, true);
+  const df = new Dataframe.Dataframe(
+    [fbs.nRows, fbs.nCols],
+    fbs.columns,
+    null,
+    new Dataframe.KeyIndex(["X", "Y"])
+  );
+  return df;
+}
+
+function reconcileSchemaCategoriesWithSummary(universe) {
+  /*
+  where we treat types as (essentially) categorical metadata, update
+  the schema with data-derived categories (in addition to those in
+  the server declared schema).
+
+  For example, boolean defined fields in the schema do not contain
+  explicit declaration of categories (nor do string fields).  In these
+  cases, add a 'categories' field to the schema so it is accessible.
+  */
+
+  _.forEach(universe.schema.annotations.obs, s => {
+    if (
+      s.type === "string" ||
+      s.type === "boolean" ||
+      s.type === "categorical"
+    ) {
+      const categories = _.union(
+        _.get(s, "categories", []),
+        _.get(universe.obsAnnotations.col(s.name).summarize(), "categories", [])
+      );
+      s.categories = categories;
+    }
+  });
+}
+
+export function createUniverseFromResponse(
+  configResponse,
+  schemaResponse,
+  annotationsObsResponse,
+  annotationsVarResponse,
+  layoutFBSResponse
+) {
+  /*
+  build & return universe from a REST 0.2 /config, /schema and /annotations/obs response
+  */
+  const { schema } = schemaResponse;
+  const universe = templateUniverse();
+
+  /* schema related */
+  universe.schema = schema;
+  universe.nObs = schema.dataframe.nObs;
+  universe.nVar = schema.dataframe.nVar;
+
+  /* annotations */
+  universe.obsAnnotations = AnnotationsFBSToDataframe(annotationsObsResponse);
+  universe.varAnnotations = AnnotationsFBSToDataframe(annotationsVarResponse);
+  /* layout */
+  universe.obsLayout = LayoutFBSToDataframe(layoutFBSResponse);
+
+  /* sanity check */
+  if (
+    universe.nObs !== universe.obsLayout.length ||
+    universe.nObs !== universe.obsAnnotations.length ||
+    universe.nVar !== universe.varAnnotations.length
+  ) {
+    throw new Error("Universe dimensionality mismatch - failed to load");
+  }
+
+  reconcileSchemaCategoriesWithSummary(universe);
+  return universe;
+}
+
+export function convertDataFBStoObject(universe, arrayBuffer) {
+  /*
+  /data/var returns a flatbuffer (FBS) as described by cellxgene/fbs/matrix.fbs
+
+  This routine converts the binary wire encoding into a JS object:
+
+  {
+    gene: Float32Array,
+    ...
+  }
+  */
+  const fbs = decodeMatrixFBS(arrayBuffer);
+  const { colIdx, columns } = fbs;
+  const result = {};
+
+  for (let c = 0; c < colIdx.length; c += 1) {
+    const varName = universe.varAnnotations.at(colIdx[c], "name");
+    result[varName] = columns[c];
+  }
+  return result;
+}

client/src/util/stateManager/world.js

@@ -0,0 +1,221 @@
+// jshint esversion: 6
+
+import _ from "lodash";
+import { layoutDimensionName, obsAnnoDimensionName } from "../nameCreators";
+import Crossfilter from "../typedCrossfilter";
+import * as Dataframe from "../dataframe";
+
+/*
+
+World is a subset of universe.   Most code should use world, and should
+(generally) not use Universe.   World contains any per-obs or per-var data
+that must be consistent acorss the app when we view/manipulate subsets
+of Universe.
+
+Private API indicated by leading underscore in key name (eg, _foo).  Anything else
+is public.
+
+Notable keys in the world object:
+
+* nObs, nVar: dimensions
+
+* schema: data schema from the server
+
+* obsAnnotations:
+
+  Dataframe containing obs annotations.  Columns are indexed by annotation
+  name (eg, 'tissue type'), and rows are indexed by the REST API obsIndex
+  (ie, the offset into the underlying server-side dataframe).
+
+  This indexing means that you can access data by _either_ the server's
+  obxIndex, or the offset into the client-side column array .  Be careful
+  to know which you want and are using.
+
+* obsLayout:
+
+  A dataframe containing the X/Y layout for all obs.  Columns are named
+  'X' and 'Y', and rows are indexed in the same way as obsAnnotation.
+
+* varData: a cache of expression columns, stored in a Dataframe.  Cache
+  managed by controls reducer.
+
+*/
+
+function templateWorld() {
+  return {
+    /* schema/version related */
+    schema: null,
+    nObs: 0,
+    nVar: 0,
+
+    /* annotations */
+    obsAnnotations: Dataframe.Dataframe.empty(),
+    varAnnotations: Dataframe.Dataframe.empty(),
+
+    /* layout of graph. Dataframe. */
+    obsLayout: Dataframe.Dataframe.empty(),
+
+    /*
+    Var data columns - subset of all data (may be empty)
+    */
+    varData: Dataframe.Dataframe.empty(null, new Dataframe.KeyIndex())
+  };
+}
+
+export function createWorldFromEntireUniverse(universe) {
+  const world = templateWorld();
+
+  /*
+  public interface follows
+  */
+
+  /* Schema related */
+  world.schema = universe.schema;
+  world.nObs = universe.nObs;
+  world.nVar = universe.nVar;
+
+  /* annotation dataframes */
+  world.obsAnnotations = universe.obsAnnotations;
+  world.varAnnotations = universe.varAnnotations;
+
+  /* layout and display characteristics dataframe */
+  world.obsLayout = universe.obsLayout;
+
+  /*
+  Var data columns - subset of all
+  */
+  world.varData = universe.varData.clone();
+
+  return world;
+}
+
+export function createWorldFromCurrentSelection(universe, world, crossfilter) {
+  const newWorld = templateWorld();
+
+  /* these don't change as only OBS are selected in our current implementation */
+  newWorld.nVar = universe.nVar;
+  newWorld.schema = universe.schema;
+  newWorld.varAnnotations = universe.varAnnotations;
+
+  /* now subset/cut obs */
+  const mask = crossfilter.allFilteredMask();
+  newWorld.obsAnnotations = world.obsAnnotations.isubsetMask(mask);
+  newWorld.obsLayout = world.obsLayout.isubsetMask(mask);
+  newWorld.nObs = newWorld.obsAnnotations.dims[0];
+
+  /*
+  Var data columns - subset of all
+  */
+  if (world.varData.isEmpty()) {
+    newWorld.varData = world.varData.clone();
+  } else {
+    newWorld.varData = world.varData.isubsetMask(mask);
+  }
+  return newWorld;
+}
+
+/*
+ Deduce the correct crossfilter dimension type from a metadata
+ schema description.
+*/
+function deduceDimensionType(attributes, fieldName) {
+  let dimensionType;
+  const { type } = attributes;
+  if (type === "string" || type === "categorical" || type === "boolean") {
+    dimensionType = "enum";
+  } else if (type === "int32") {
+    dimensionType = Int32Array;
+  } else if (type === "float32") {
+    dimensionType = Float32Array;
+  } else {
+    /*
+    Currently not supporting boolean and categorical types.
+    */
+    console.error(
+      `Warning - REST API returned unknown metadata schema (${type}) for field ${fieldName}.`
+    );
+    // skip it - we don't know what to do with this type
+  }
+  return dimensionType;
+}
+
+/*
+  Return a crossfilter dimension for the specified world & named gene.
+
+  NOTE: this assumes that the expression data was already loaded,
+  by calling an appropriate action creator.
+
+  Caller needs to *save* this dimension somewhere for it to be later used.
+  Dimension must be destroyed by calling dimension.dispose()
+  when it is no longer needed
+  (it will not be garbage collected without this call)
+*/
+export function createVarDataDimension(world, crossfilter, name) {
+  return crossfilter.dimension(
+    Crossfilter.ScalarDimension,
+    world.varData.col(name).asArray(),
+    Float32Array
+  );
+}
+
+export function createObsDimensionMap(crossfilter, world) {
+  /*
+  create and return a crossfilter dimension for every obs annotation
+  for which we have a supported type.
+  */
+  const { schema, obsLayout, obsAnnotations } = world;
+
+  // Create a crossfilter dimension for all obs annotations *except* 'name'
+  const dimensionMap = _(schema.annotations.obs)
+    .filter(anno => anno.name !== "name")
+    .transform((result, anno) => {
+      const dimType = deduceDimensionType(anno, anno.name);
+      const colData = obsAnnotations.col(anno.name).asArray();
+      if (dimType === "enum") {
+        result[obsAnnoDimensionName(anno.name)] = crossfilter.dimension(
+          Crossfilter.EnumDimension,
+          colData
+        );
+      } else if (dimType) {
+        result[obsAnnoDimensionName(anno.name)] = crossfilter.dimension(
+          Crossfilter.ScalarDimension,
+          colData,
+          dimType
+        );
+      } // else ignore the annotation
+    }, {})
+    .value();
+
+  /*
+  Add crossfilter dimensions allowing filtering on layout
+  */
+  dimensionMap[layoutDimensionName("XY")] = crossfilter.dimension(
+    Crossfilter.SpatialDimension,
+    obsLayout.col("X").asArray(),
+    obsLayout.col("Y").asArray()
+  );
+
+  return dimensionMap;
+}
+
+export function worldEqUniverse(world, universe) {
+  return world.obsAnnotations === universe.obsAnnotations;
+}
+
+export function getSelectedByIndex(crossfilter) {
+  /*
+  return array of obsIndex, containing all selected obs/cells.
+  */
+  const selected = crossfilter.allFilteredMask(); // array of bool-ish
+  const keys = crossfilter.data.rowIndex.keys(); // row keys, aka universe rowIndex
+
+  const set = new Int32Array(selected.length);
+  let numElems = 0;
+  for (let i = 0, l = selected.length; i < l; i += 1) {
+    if (selected[i]) {
+      set[numElems] = keys[i];
+      numElems += 1;
+    }
+  }
+  return new Int32Array(set.buffer, 0, numElems);
+}

client/src/util/stateManager/worldUtil.js

@@ -0,0 +1,75 @@
+/* eslint-disable import/prefer-default-export */
+import _ from "lodash";
+
+/*
+Various utility functions operating on World/Universe
+*/
+
+/*
+Count unique category values, binning first by dim1 then by dim2
+Return:
+
+Map {
+   dim1_val1: Map {
+      dim2_val1: number,
+      dim2_val2: number,
+      ...
+   },
+   ...
+}
+
+Parameters are:
+- dim1: dimension 1 name/label
+- dim2: dimension 2 name/label
+- df: dataframe containing dim1 and dim2 on the column axis
+
+*/
+function _countCategoryValues2D(dim1, dim2, df) {
+  const dimMap = new Map();
+  const col1 = df.col(dim1) ? df.col(dim1).asArray() : null;
+  const col2 = df.col(dim2) ? df.col(dim2).asArray() : null;
+  if (!col1 || !col2) {
+    return dimMap;
+  }
+
+  for (let r = 0, l = df.length; r < l; r += 1) {
+    const val1 = col1[r];
+    const val2 = col2[r];
+    let d2Map = dimMap.get(val1);
+    if (d2Map === undefined) {
+      d2Map = new Map();
+      dimMap.set(val1, d2Map);
+    }
+    let curCount = d2Map.get(val2);
+    if (curCount === undefined) {
+      curCount = 0;
+    }
+    d2Map.set(val2, curCount + 1);
+  }
+  return dimMap;
+}
+
+let __worldUtilMemoId__ = 0;
+function _memoizedId(x) {
+  if (!x.__worldUtilMemoId__) {
+    __worldUtilMemoId__ += 1;
+    x.__worldUtilMemoId__ = __worldUtilMemoId__;
+  }
+  return x.__worldUtilMemoId__;
+}
+function _countCategoryValues2DResolver(...args) {
+  const id = args[0] + args[1] + _memoizedId(args[2]);
+  return id;
+}
+
+export const countCategoryValues2D = _.memoize(
+  _countCategoryValues2D,
+  _countCategoryValues2DResolver
+);
+
+/*
+Clear any cached data within WorldUtil caches, eg, memoized functions
+*/
+export function clearCaches() {
+  countCategoryValues2D.cache.clear();
+}

client/src/util/typedCrossfilter/bitArray.js

@@ -0,0 +1,265 @@
+// jshint esversion: 6
+/* eslint no-bitwise: "off" */
+
+// BitArray is a 2D bitarray with size [length, nBitWidth].
+// Each bit is referred to as a `dimension`.  Dimensions may be
+// dynamically allocated and deallocated.  The overall length
+// of the BitArray is fixed at creation time (for simplicity).
+//
+// Organization of the bitarray is dimension-major.  As dimensions
+// are added, the underlying store is grown 32 bits at a time.
+// NOTE: currently does not deallocate / shrink.
+//
+// Primary operations on the BitArray are:
+//    - set & clear dimension
+//    - test dimension
+//    - various performance or convenience operations to optimize bulk ops
+//
+// The underlying data structure uses TypedArrays for performance.
+//
+class BitArray {
+  constructor(length) {
+    // Initially allocate a 32 bit wide array.  allocDimension() will expand
+    // as necessary.
+    //
+    // Int32Array is (counterintuitively) used to accomadate JS numeric casting
+    // (to/from primitive number type).
+    //
+
+    // Fixed for the life of this object.
+    this.length = length;
+
+    // Bitarray width.  width is always greater than dimensionCount/32.
+    this.width = 1; // underlying number of 32 bit arrays
+    this.dimensionCount = 0; // num allocated dimensions
+
+    this.bitmask = new Int32Array(this.width); // dimension allocation mask
+    this.bitarray = new Int32Array(this.width * this.length);
+  }
+
+  // Return the number of records that are selected, ie, have a one bit in
+  // all allocated dimensions.
+  //
+  selectionCount() {
+    return this.countAllOnes();
+  }
+
+  // Count all records that have a 'one' bit in allocated dimensions.
+  //
+  countAllOnes() {
+    let count = 0;
+    const { bitarray, length, width } = this;
+    if (width === 1) {
+      // special case, width === 1, for performance
+      const bitmask = this.bitmask[0];
+      for (let l = 0; l < length; l += 1) {
+        if (bitarray[l] === bitmask) {
+          count += 1;
+        }
+      }
+    } else {
+      const { bitmask } = this;
+      for (let l = 0; l < length; l += 1) {
+        let dimensionsSet = 0;
+        for (let w = 0; w < width; w += 1) {
+          if (bitarray[w * length + l] === bitmask[w]) {
+            dimensionsSet += 1;
+          }
+        }
+        if (dimensionsSet === width) {
+          count += 1;
+        }
+      }
+    }
+    return count;
+  }
+
+  // count trailing zeros - hard to do fast in JS!
+  // https://en.wikipedia.org/wiki/Find_first_set#CTZ
+  static ctz(av) {
+    let c = 32;
+    let v = av;
+    v &= -v; // isolate lowest non-zero bit
+    if (v) c -= 1;
+    if (v & 0x0000ffff) c -= 16;
+    if (v & 0x00ff00ff) c -= 8;
+    if (v & 0x0f0f0f0f) c -= 4;
+    if (v & 0x33333333) c -= 2;
+    if (v & 0x55555555) c -= 1;
+    return c;
+  }
+
+  // find a free dimension.  Return undefined if none
+  _findFreeDimension() {
+    let dim;
+    for (let col = 0; col < this.width; col += 1) {
+      const lowestZeroBit = ~this.bitmask[col] & -~this.bitmask[col];
+      if (lowestZeroBit) {
+        this.bitmask[col] |= lowestZeroBit;
+        dim = 32 * col + BitArray.ctz(lowestZeroBit);
+      }
+    }
+    return dim;
+  }
+
+  // allocate and return the dimension ID (bit position)
+  //
+  allocDimension() {
+    let dim = this._findFreeDimension();
+
+    // if we did not find free dimension, expand the bitarray.
+    if (dim === undefined) {
+      this.width += 1;
+
+      const biggerBitArray = new Int32Array(this.width * this.length);
+      biggerBitArray.set(this.bitarray);
+      this.bitarray = biggerBitArray;
+
+      const biggerBitmask = new Int32Array(this.width);
+      biggerBitmask.set(this.bitmask);
+      this.bitmask = biggerBitmask;
+
+      dim = this._findFreeDimension();
+    }
+
+    this.dimensionCount += 1;
+    return dim;
+  }
+
+  // free a dimension for later use.  MUST deselect the dimension, as other
+  // code assume the column will be zero valued.
+  //
+  freeDimension(dim) {
+    // all selection tests assume unallocated dimensions are zero valued.
+    this.deselectAll(dim);
+    const col = dim >>> 5;
+    this.bitmask[col] &= ~(1 << dim % 32);
+    this.dimensionCount -= 1;
+  }
+
+  // return true if this index is selected in ALL dimensions.
+  //
+  isSelected(index) {
+    const { width, length, bitarray } = this;
+
+    for (let w = 0; w < width; w += 1) {
+      const bitmask = this.bitmask[w];
+      if (!bitmask || bitarray[w * length + index] !== bitmask) return false;
+    }
+    return true;
+  }
+
+  // return true if this index is selected in ALL dimensions IGNORING dim
+  //
+  isSelectedIgnoringDim(index, dim) {
+    const ignoreOffset = dim >>> 5;
+    const ignoreMask = ~(1 << dim % 32);
+
+    const { width, length, bitarray } = this;
+
+    for (let w = 0; w < width; w += 1) {
+      const bitmask = this.bitmask[w];
+      if (w === ignoreOffset) {
+        if (
+          bitmask &&
+          (bitarray[w * length + index] & ignoreMask) !== (bitmask & ignoreMask)
+        ) {
+          return false;
+        }
+      } else if (bitmask && bitarray[w * length + index] !== bitmask) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  // select index on dimension
+  //
+  selectOne(dim, index) {
+    const col = dim >>> 5;
+    const before = this.bitarray[col * this.length + index];
+    const after = before | (1 << dim % 32);
+    this.bitarray[col * this.length + index] = after;
+  }
+
+  // deselect index on dimension
+  //
+  deselectOne(dim, index) {
+    const col = dim >>> 5;
+    const before = this.bitarray[col * this.length + index];
+    const after = before & ~(1 << dim % 32);
+    this.bitarray[col * this.length + index] = after;
+  }
+
+  // select all indices on dimension.
+  //
+  selectAll(dim) {
+    const col = dim >> 5;
+    const one = 1 << dim % 32;
+    for (let i = col * this.length, len = i + this.length; i < len; i += 1) {
+      this.bitarray[i] |= one;
+    }
+  }
+
+  // deselect all indices on dimension
+  //
+  deselectAll(dim) {
+    const col = dim >> 5;
+    const zero = ~(1 << dim % 32);
+    for (let i = col * this.length, len = i + this.length; i < len; i += 1) {
+      this.bitarray[i] &= zero;
+    }
+  }
+
+  // select range of indices on a dimension, indirect through a sort map.
+  // Indirect functions are used to map between sort and natural order.
+  //
+  selectIndirectFromRange(dim, indirect, range) {
+    const col = dim >>> 5;
+    const first = range[0];
+    const last = range[1];
+    const one = 1 << dim % 32;
+    const offset = col * this.length;
+    for (let i = first; i < last; i += 1) {
+      this.bitarray[offset + indirect[i]] |= one;
+    }
+  }
+
+  // deselect range of indices on a dimension, indirect through a sort map.
+  //
+  deselectIndirectFromRange(dim, indirect, range) {
+    const col = dim >>> 5;
+    const first = range[0];
+    const last = range[1];
+    const zero = ~(1 << dim % 32);
+    const offset = col * this.length;
+    for (let i = first; i < last; i += 1) {
+      this.bitarray[offset + indirect[i]] &= zero;
+    }
+  }
+
+  // Fill the array with selected|deselected value based upon the
+  // current selection state.
+  //
+  fillBySelection(result, selectedValue, deselectedValue) {
+    // special case (width === 1) for performance
+    if (this.width === 1) {
+      const { bitmask, bitarray } = this;
+      const mask = bitmask[0];
+      if (!mask) {
+        result.fill(deselectedValue);
+      } else {
+        for (let i = 0, len = this.length; i < len; i += 1) {
+          result[i] = bitarray[i] === mask ? selectedValue : deselectedValue;
+        }
+      }
+    } else {
+      for (let i = 0, len = this.length; i < len; i += 1) {
+        result[i] = this.isSelected(i) ? selectedValue : deselectedValue;
+      }
+    }
+    return result;
+  }
+}
+
+export default BitArray;

client/src/util/typedCrossfilter/index.js

@@ -0,0 +1,791 @@
+// jshint esversion: 6
+
+/*
+Typedarray Crossfilter - a re-implementation of a subset of crossfilter, with
+time/space optimizations predicated upon the following assumptions:
+  - dimensions are uniformly typed, and all values must be of that type
+  - dimension values must be a primitive type (int, float, string).  Arrays
+    or other complex types not supported.
+  - dimension creation requires call-provided type declaration
+  - no support for adding/removing data to an existing crossfilter.  If you
+    want to do that, you have to create the new crossfilter, using the new
+    data, from scratch.
+
+The actual backing store for a dimension is a TypedArray, enabling significant
+performance improvements over the original crossfilter.
+
+There are also a handful of new methods, primarily to take advantage of the
+performance (eg, crossfilter.fillBySelection)
+
+Helpful documents (this module tries to follow the original API as much
+as is feasable):
+    https://github.com/square/crossfilter/
+    http://square.github.io/crossfilter/
+
+There is also a newer, community supported fork of crossfilter, with a
+more complex API.  In a few cases, elements of that API were incorporated.
+    https://github.com/square/crossfilter/
+
+*/
+// XXX replace
+import { polygonContains } from "d3";
+
+import PositiveIntervals from "./positiveIntervals";
+import BitArray from "./bitArray";
+import {
+  makeSortIndex,
+  lowerBound,
+  lowerBoundIndirect,
+  upperBoundIndirect
+} from "./util";
+
+function isArrayOrTypedArray(x) {
+  return (
+    Array.isArray(x) ||
+    (ArrayBuffer.isView(x) &&
+      Object.prototype.toString.call(x) !== "[object DataView]")
+  );
+}
+
+class NotImplementedError extends Error {
+  constructor(...params) {
+    super(...params);
+
+    // Maintains proper stack trace for where our error was thrown (only available on V8)
+    if (Error.captureStackTrace) {
+      Error.captureStackTrace(this, NotImplementedError);
+    }
+  }
+}
+
+class TypedCrossfilter {
+  constructor(data) {
+    /*
+    Typically, data is one of:
+      - Array of objects/records
+      - Dataframe (util/dataframe)
+    */
+    this.data = data;
+
+    // filters: array of { id, dimension }
+    this.filters = [];
+    this.selection = new BitArray(data.length);
+    this.updateTime = 0;
+  }
+
+  size() {
+    return this.data.length;
+  }
+
+  all() {
+    return this.data;
+  }
+
+  /*
+  Create a crossfilter dimension, upon which filtering (subselection) can
+  be done.  Each dimension is typed, and has a particular set of filtering
+  semantics.
+    * ScalarDimension - backed by TypedArray values, supporting filtering
+      by value (within a value range, or one or more exact values)
+    * EnumDimension - backed by an enumeration (eg, strings, bools), filtering
+      by one or more enum categories.
+    * SpatialDimension - backed by 2D points, filter by containment within
+      various shapes (currently supports within Rectangle and within Polygon).
+  Call this method to create a dimension, passing arguments appropriate for
+  the dimension constructor.
+  */
+  dimension(DimensionType, ...rest) {
+    const id = this.selection.allocDimension();
+    const dim = new DimensionType(this, id, ...rest);
+    this.filters.push({ id, dim });
+    dim.filterAll();
+    return dim;
+  }
+
+  _freeDimension(id) {
+    this.selection.freeDimension(id);
+    this.filters = this.filters.filter(f => f._id !== id);
+  }
+
+  // return array of all records that are selected/filtered
+  // by all dimensions.
+  allFiltered() {
+    const { data, selection } = this;
+    if (Array.isArray(data)) {
+      const res = [];
+      for (let i = 0, len = data.length; i < len; i += 1) {
+        if (selection.isSelected(i)) {
+          res.push(data[i]);
+        }
+      }
+      return res;
+    }
+    /* else, Dataframe-like */
+    return data.isubsetMask(this.allFilteredMask());
+  }
+
+  // return Uint8array containing selection state (truthy/falsey) for each record.
+  //
+  allFilteredMask() {
+    return this.selection.fillBySelection(
+      new Uint8Array(this.data.length),
+      1,
+      0
+    );
+  }
+
+  countFiltered() {
+    return this.selection.selectionCount();
+  }
+
+  isElementFiltered(i) {
+    return this.selection.isSelected(i);
+  }
+
+  // fill array with one of two values, based upon selection state
+  fillByIsFiltered(array, selectedValue, deselectedValue) {
+    return this.selection.fillBySelection(
+      array,
+      selectedValue,
+      deselectedValue
+    );
+  }
+}
+
+// Base dimension type - not exported.
+class _Dimension {
+  constructor(xfltr, id) {
+    this.crossfilter = xfltr;
+    this._id = id;
+    this.groups = [];
+  }
+
+  dispose() {
+    this.crossfilter._freeDimension(this._id);
+    return this;
+  }
+
+  id() {
+    return this._id;
+  }
+
+  _filterUpdate() {
+    this.crossfilter.updateTime += 1;
+  }
+}
+
+// Scalar dimension type - value must be a scalar type (eg, int, float),
+// and value array must be a TypedArray.
+//
+class ScalarDimension extends _Dimension {
+  constructor(xfltr, id, value, ValueArrayType) {
+    super(xfltr, id);
+
+    // current selection filter, expressed as PostiveIntervals.
+    this.currentFilter = [];
+
+    // Two modes - caller can provide a pre-created value array,
+    // or a map function which will create it.
+    let array;
+    if (value instanceof ValueArrayType) {
+      // user has provided the final typed array - just use it
+      if (value.length !== this.crossfilter.data.length) {
+        throw new RangeError(
+          "ScalarDimension values length must equal crossfilter data record count"
+        );
+      }
+      array = value;
+    } else if (value instanceof Function) {
+      // Create value array from user-provided map function.
+      array = this._createValueArray(
+        value,
+        new ValueArrayType(this.crossfilter.data.length)
+      );
+    } else if (isArrayOrTypedArray(value)) {
+      // Create value array from user-provided array.  Typically used
+      // only by enumerated dimensions
+      array = this._createValueArray(
+        i => value[i],
+        new ValueArrayType(this.crossfilter.data.length)
+      );
+    } else {
+      throw new NotImplementedError(
+        "dimension value must be function or value array type"
+      );
+    }
+    this.value = array;
+
+    // create sort index
+    this.index = makeSortIndex(array);
+  }
+
+  _createValueArray(value, array) {
+    // create dimension value array
+    const { data } = this.crossfilter;
+    const len = data.length;
+    const larray = array;
+    for (let i = 0; i < len; i += 1) {
+      larray[i] = value(i, data);
+    }
+    return larray;
+  }
+
+  // Argument is an array of intervals indicating records newly selected/filtered
+  //
+  _updateFilters(newFilter) {
+    const cNewFilter = PositiveIntervals.canonicalize(newFilter);
+
+    const adds = PositiveIntervals.difference(cNewFilter, this.currentFilter);
+    const dels = PositiveIntervals.difference(this.currentFilter, cNewFilter);
+
+    this.crossfilter.filters.forEach(f =>
+      f.dim.groups.forEach(grp => grp._updateReduceDel(this, dels))
+    );
+
+    dels.forEach(interval =>
+      this.crossfilter.selection.deselectIndirectFromRange(
+        this._id,
+        this.index,
+        interval
+      )
+    );
+
+    adds.forEach(interval =>
+      this.crossfilter.selection.selectIndirectFromRange(
+        this._id,
+        this.index,
+        interval
+      )
+    );
+
+    this.crossfilter.filters.forEach(f =>
+      f.dim.groups.forEach(grp => grp._updateReduceAdd(this, adds))
+    );
+
+    this.currentFilter = cNewFilter;
+    this._filterUpdate();
+  }
+
+  // filter by value - exact match
+  filterExact(value) {
+    const newFilter = [
+      lowerBoundIndirect(this.value, this.index, value, 0, this.value.length),
+      upperBoundIndirect(this.value, this.index, value, 0, this.value.length)
+    ];
+    if (newFilter[0] <= newFilter[1]) {
+      this._updateFilters([newFilter]);
+    } else {
+      this._updateFilters([]);
+    }
+    return this;
+  }
+
+  // filter by a set of values, eg. enum.
+  filterEnum(values) {
+    const newFilter = [];
+    for (let v = 0, len = values.length; v < len; v += 1) {
+      const intv = [
+        lowerBoundIndirect(
+          this.value,
+          this.index,
+          values[v],
+          0,
+          this.value.length
+        ),
+        upperBoundIndirect(
+          this.value,
+          this.index,
+          values[v],
+          0,
+          this.value.length
+        )
+      ];
+      if (intv[0] <= intv[1]) newFilter.push(intv);
+    }
+    this._updateFilters(newFilter);
+    return this;
+  }
+
+  // filter by value range [lo, hi)
+  // lo: inclusive, hi: exclusive
+  filterRange(range) {
+    const newFilter = [];
+    const intv = [
+      lowerBoundIndirect(
+        this.value,
+        this.index,
+        range[0],
+        0,
+        this.value.length
+      ),
+      upperBoundIndirect(this.value, this.index, range[1], 0, this.value.length)
+    ];
+    if (intv[0] < intv[1]) newFilter.push(intv);
+    this._updateFilters(newFilter);
+    return this;
+  }
+
+  // select all - equivalent of selecting all in this dimension
+  filterAll() {
+    this._updateFilters([[0, this.value.length]]);
+    return this;
+  }
+
+  // select none
+  filterNone() {
+    this._updateFilters([]);
+  }
+
+  // return top k records, starting with offset, in descending order.
+  // Order is this dimension's sort order
+  top(k, offset = 0) {
+    const { data, selection } = this.crossfilter;
+    const { index } = this;
+    const len = index.length;
+    const ret = [];
+    let i = 0;
+    let skip = 0;
+    let found = 0;
+
+    // skip up to offset records
+    for (i = len - 1; i >= 0 && skip < offset; i -= 1) {
+      if (selection.isSelected(index[i])) {
+        skip += 1;
+      }
+    }
+
+    // grab up to k records
+    for (; i >= 0 && found < k; i -= 1) {
+      if (selection.isSelected(index[i])) {
+        ret.push(data[index[i]]);
+        found += 1;
+      }
+    }
+
+    return ret;
+  }
+
+  // return bottom k records, starting with offset, in ascending order.
+  // Order is this dimension's sort order
+  bottom(k, offset = 0) {
+    const { data, selection } = this.crossfilter;
+    const { index } = this;
+    const len = index.length;
+    const ret = [];
+    let skip = 0;
+    let found = 0;
+    let i = 0;
+
+    // skip up to offset records
+    for (i = 0; i < len && skip < offset; i += 1) {
+      if (selection.isSelected(index[i])) {
+        skip += 1;
+      }
+    }
+
+    // grab up to k records
+    for (; i < len && found < k; i += 1) {
+      if (selection.isSelected(index[i])) {
+        ret.push(data[index[i]]);
+        found += 1;
+      }
+    }
+
+    return ret;
+  }
+
+  group(groupValue) {
+    const grp = new ScalarGroup(groupValue, this.value.constructor, this);
+    this.groups.push(grp);
+    return grp;
+  }
+
+  _freeGroup(group) {
+    this.groups = this.groups.filter(e => e !== group);
+  }
+}
+
+// Ordered enumeration - supports any sortable enumerable type, eg,
+// strings, which can be mapped into an fixed numeric range [0..n).
+//
+class EnumDimension extends ScalarDimension {
+  constructor(xfltr, id, value) {
+    super(xfltr, id, value, Uint32Array);
+  }
+
+  _createValueArray(value, array) {
+    const { data } = this.crossfilter;
+    const len = data.length;
+    const larray = array;
+
+    // create enumeration table - mapping between the value
+    // and the enum.
+    const s = new Set();
+    for (let i = 0; i < len; i += 1) {
+      s.add(value(i, data));
+    }
+    this.enumIndex = Array.from(s);
+    this.enumIndex.sort();
+
+    // create dimension value array
+    const enumLen = this.enumIndex.length;
+    for (let i = 0; i < len; i += 1) {
+      const v = value(i, data);
+      const e = lowerBound(this.enumIndex, v, 0, enumLen);
+      larray[i] = e;
+    }
+    return larray;
+  }
+
+  filterExact(value) {
+    return super.filterExact(
+      lowerBound(this.enumIndex, value, 0, this.enumIndex.length)
+    );
+  }
+
+  filterEnum(values) {
+    return super.filterEnum(
+      values.map(v => lowerBound(this.enumIndex, v, 0, this.enumIndex.length))
+    );
+  }
+
+  filterRange(range) {
+    return super.filterEnum(
+      range.map(v => lowerBound(this.enumIndex, v, 0, this.enumIndex.length))
+    );
+  }
+
+  group(groupValue) {
+    const grp = new EnumGroup(groupValue, this.value.constructor, this);
+    this.groups.push(grp);
+    return grp;
+  }
+}
+
+/*
+Super simple 2D spatial dimension, supporting basic "filter within"
+operations.
+*/
+class SpatialDimension extends _Dimension {
+  constructor(xfltr, id, X, Y) {
+    super(xfltr, id);
+
+    if (X.length !== Y.length && X.length !== this.crossfilter.data.length) {
+      throw new RangeError(
+        "SpatialDimension values must have same dimensionality as crossfilter"
+      );
+    }
+    this.X = X;
+    this.Y = Y;
+
+    this.Xindex = makeSortIndex(X);
+    this.Yindex = makeSortIndex(Y);
+  }
+
+  filterAll() {
+    this.crossfilter.selection.selectAll(this._id);
+    this._filterUpdate();
+  }
+
+  filterNone() {
+    this.crossfilter.selection.deselectAll(this._id);
+    this._filterUpdate();
+  }
+
+  /*
+  this could be smarter, but we don't currently use it...
+  */
+  filterWithinRect(northwest, southeast) {
+    const [x0, y0] = northwest;
+    const [x1, y1] = southeast;
+    const { X, Y } = this;
+    const seln = this.crossfilter.selection;
+    const { _id } = this;
+    seln.deselectAll(_id);
+    for (let i = 0, l = this.X.length; i < l; i += 1) {
+      const x = X[i];
+      const y = Y[i];
+      if (x0 <= x && x < x1 && y0 <= y && y < y1) {
+        seln.selectOne(_id, i);
+      }
+    }
+    this._filterUpdate();
+  }
+
+  /*
+  Relatively brute force filter by polygon.   Polygon is array of points, where
+  each point is [x,y].   Eg, [[x0,y0], [x1,y1], ...].
+
+  Currently uses d3.polygonContains() to test for polygon inclusion, which itself
+  uses a ray casting (crossing number) algorithm.  There are a series of optimizations
+  to make this faster:
+    * first sliced by X or Y, using an index on the axis
+    * then the polygon bounding box is used for trivial rejection
+    * then the polygon test is applied
+  */
+  filterWithinPolygon(polygon) {
+    /* return bounding box of the polygon */
+    function polygonBoundingBox(pg) {
+      let minX = Number.MAX_VALUE;
+      let minY = Number.MAX_VALUE;
+      let maxX = Number.MIN_VALUE;
+      let maxY = Number.MIN_VALUE;
+      for (let i = 0, l = pg.length; i < l; i += 1) {
+        const p = pg[i];
+        const x = p[0];
+        const y = p[1];
+        if (x < minX) minX = x;
+        if (y < minY) minY = y;
+        if (x > maxX) maxX = x;
+        if (y > maxY) maxY = y;
+      }
+      return [minX, minY, maxX, maxY];
+    }
+
+    const [minX, minY, maxX, maxY] = polygonBoundingBox(polygon);
+    const { X, Y } = this;
+    let slice;
+    let index;
+    if (maxY - minY > maxX - minX) {
+      slice = [
+        lowerBoundIndirect(X, this.Xindex, minX, 0, X.length),
+        upperBoundIndirect(X, this.Xindex, maxX, 0, X.length)
+      ];
+      index = this.Xindex;
+    } else {
+      slice = [
+        lowerBoundIndirect(Y, this.Yindex, minY, 0, Y.length),
+        upperBoundIndirect(Y, this.Yindex, maxY, 0, Y.length)
+      ];
+      index = this.Yindex;
+    }
+
+    const seln = this.crossfilter.selection;
+    const { _id } = this;
+    const testWithin = polygonContains; // d3.polygonContains()
+    seln.deselectAll(_id);
+
+    for (let i = slice[0], e = slice[1]; i < e; i += 1) {
+      const rid = index[i];
+      const x = X[rid];
+      const y = Y[rid];
+      if (
+        minX <= x &&
+        x < maxX &&
+        minY <= y &&
+        y < maxY &&
+        testWithin(polygon, [x, y])
+      ) {
+        seln.selectOne(_id, rid);
+      }
+    }
+    this._filterUpdate();
+  }
+}
+
+// Groups!  Map/reduce
+//
+class ScalarGroup {
+  constructor(groupValue, groupValueType, dimension) {
+    // parent dimension
+    this.dimension = dimension;
+
+    // generate group names from dimension values
+    this.mapValue = this.constructor._map(
+      groupValue,
+      groupValueType,
+      dimension
+    );
+
+    // group index is mapping from data record index to group index
+    this.groupIndex = new Uint32Array(dimension.crossfilter.data.length);
+
+    // default to counting
+    this.reduceCount();
+
+    // Creates this.groups
+    this._reduce();
+  }
+
+  // internal support function - map all dimension values to group values.
+  //
+  static _map(groupValue, GroupValueType, dimension) {
+    // groupValue is optional.  Defaults to identity.  Used to perform
+    // initial map operation.
+    //
+    // identity: save some memory...
+    if (groupValue === undefined) return dimension.value;
+
+    const data = dimension.value;
+    const len = data.length;
+    const mapValue = new GroupValueType(dimension.value.length);
+    for (let i = 0; i < len; i += 1) {
+      mapValue[i] = groupValue(data[i]);
+    }
+    return mapValue;
+  }
+
+  // Update the group reduction incrementally.  Called when *any* dimension filter
+  // changes.  Guaranteed to be called AFTER the crossfilter is updated.
+  //
+  // Arguments:
+  //  * dim: the dimension that is changing
+  //  * intv: interval list of newly selected values on `dim` (adds)
+  //
+  _updateReduceAdd(dim, intv) {
+    // ignore updates to self, as we don't reduce inclusive of our filter
+    if (dim === this.dimension || intv.length === 0) return;
+
+    // Each item in the range was just added to `dim`.  It was NOT previously
+    // selected - reduceAdd if it is now selected.
+    const { data, selection } = this.dimension.crossfilter;
+    intv.forEach(rng => {
+      for (let r = rng[0]; r < rng[1]; r += 1) {
+        const i = dim.index[r];
+        if (selection.isSelectedIgnoringDim(i, this.dimension.id())) {
+          const group = this.groups[this.groupIndex[i]];
+          group.value = this.reduceAdd(group.value, data[i]);
+        }
+      }
+    });
+  }
+
+  // Update the group reduction incrementally.  Called when *any* dimension filter
+  // changes.   Guaranteed to be called BEFORE the crossfilter is updated.
+  //
+  // Arguments:
+  //  * dim: the dimension that is changing
+  //  * intv: interval list of previously selected values on `dim` (dels)
+  //
+  _updateReduceDel(dim, intv) {
+    // ignore updates to self, as we don't reduce inclusive of our filter
+    if (dim === this.dimension || intv.length === 0) return;
+
+    // Each item in the range will be remved from `dim`.  reduceRemove if it
+    // is currently selected.
+    const { data, selection } = this.dimension.crossfilter;
+    intv.forEach(rng => {
+      for (let r = rng[0]; r < rng[1]; r += 1) {
+        const i = dim.index[r];
+        if (selection.isSelectedIgnoringDim(i, this.dimension.id())) {
+          const group = this.groups[this.groupIndex[i]];
+          group.value = this.reduceRemove(group.value, data[i]);
+        }
+      }
+    });
+  }
+
+  // Reduce the entire data set, creating both the group index and the
+  // groups data.
+  //
+  _reduce() {
+    const { dimension } = this;
+    const { data } = dimension.crossfilter;
+
+    // Create groups
+    const groupNames = new Set(this.mapValue);
+    this.groups = [];
+    const groupIndexByName = {};
+    groupNames.forEach(name => {
+      this.groups.push({ key: name, value: this.reduceInitial() });
+      groupIndexByName[name] = this.groups.length - 1;
+    });
+
+    // Create groupIndex - index map between data record index and group index
+    for (let i = 0, len = this.mapValue.length; i < len; i += 1) {
+      this.groupIndex[i] = groupIndexByName[this.mapValue[i]];
+    }
+
+    // reduce all filtered records, IGNORING the current dimension's filter
+    const { selection } = dimension.crossfilter;
+    for (let i = 0, len = data.length; i < len; i += 1) {
+      if (selection.isSelectedIgnoringDim(i, dimension.id())) {
+        const group = this.groups[this.groupIndex[i]];
+        group.value = this.reduceAdd(group.value, data[i]);
+      }
+    }
+  }
+
+  dispose() {
+    this.dimension._freeGroup(this);
+    return this;
+  }
+
+  // return number of distinct values in the group, independent of any filters.
+  //
+  size() {
+    return this.groups.length;
+  }
+
+  // Set the reduce functions and return the grouping.
+  //
+  reduce(add, remove, initial) {
+    this.reduceAdd = add;
+    this.reduceRemove = remove;
+    this.reduceInitial = initial;
+    this._reduce();
+    return this;
+  }
+
+  // set the reduce functions to count records.
+  reduceCount() {
+    return this.reduce(p => p + 1, p => p - 1, () => 0);
+  }
+
+  // set the reduce functions to sum records using specified value accessor.
+  //
+  reduceSum(value) {
+    return this.reduce((p, v) => p + value(v), (p, v) => p - value(v), () => 0);
+  }
+
+  all() {
+    const res = [...this.groups];
+    res.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
+    return res;
+  }
+}
+
+class EnumGroup extends ScalarGroup {
+  static _map(groupValue, groupValueType, dimension) {
+    // groupValue is optional.  Defaults to identity.  Used to perform
+    // initial map operation.
+    //
+    // identity: save some memory
+    if (groupValue === undefined) return dimension.value;
+
+    // non-identity mapping unsupported for EnumDimension/EnumGroup.
+    // XXX: this could be implemented, but would require another index
+    // array to map from the group names/keys back to the dimension values.
+    // With this, we just rely on the dimensions `enumIndex` to map from
+    // enumeration value to the record.
+    throw new NotImplementedError("enumerated group mapping not implemented");
+  }
+
+  all() {
+    const res = [];
+    this.groups.forEach(e =>
+      res.push({
+        // XXX: assumes identity group map - see comment in _map()
+        key: this.dimension.enumIndex[e.key],
+        value: e.value
+      })
+    );
+    res.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
+    return res;
+  }
+}
+
+// Wrapper for backwards compat with crossfilter.
+//
+function crossfilter(data) {
+  return new TypedCrossfilter(data);
+}
+
+crossfilter.PositiveIntervals = PositiveIntervals;
+crossfilter.BitArray = BitArray;
+crossfilter.TypedCrossfilter = TypedCrossfilter;
+crossfilter.ScalarDimension = ScalarDimension;
+crossfilter.EnumDimension = EnumDimension;
+crossfilter.SpatialDimension = SpatialDimension;
+
+export default crossfilter;

client/src/util/typedCrossfilter/positiveIntervals.js

@@ -0,0 +1,127 @@
+// jshint esversion: 6
+
+// Interval operations - very simple version of interval set relationship
+// operators.   An interval is a multi-interval list of [min, max),
+// where min and max are mandatory.   Constraints:
+//    * min <= max, min >= 0
+//    * empty interval groups are OK, ie, []
+//    * Legal intervals:   [], [ [0, 1], ... ]
+//    * Not legal:   [ [] ]
+//
+// All intervals are represented by simple JS arrays/numbers.
+//
+// Code assumes intervals have a low cardinality; many operations are done
+// with a brute force scan.   Little attempt to reduce GC pressure.
+//
+class PositiveIntervals {
+  // Canonicalize - ensure that:
+  //  1. no overlapping intervals
+  //  2. sorted in order of interval min.
+  //
+  static canonicalize(A) {
+    if (A.length <= 1) return A;
+    const copy = A.slice();
+    copy.sort((a, b) => a[0] - b[0]);
+    const res = [];
+    res.push(copy[0]);
+    for (let i = 1, len = copy.length; i < len; i += 1) {
+      if (copy[i][0] > res[res.length - 1][1]) {
+        // non-overlapping, add to result
+        res.push(copy[i]);
+      } else if (copy[i][1] > res[res.length - 1][1]) {
+        // merge this into previous
+        res[res.length - 1][1] = copy[i][1]; // eslint-disable-line prefer-destructuring
+      }
+    }
+    return res;
+  }
+
+  // Return interval with values belonging to both A and B. Essentially
+  // a set union operation.
+  //
+  static union(A, B) {
+    return PositiveIntervals.canonicalize([...A, ...B]);
+  }
+
+  static _flatten(A, B) {
+    const points = []; /* point, A, start */
+    for (let a = 0; a < A.length; a += 1) {
+      points.push([A[a][0], true, true]);
+      points.push([A[a][1], true, false]);
+    }
+    for (let b = 0; b < B.length; b += 1) {
+      points.push([B[b][0], false, true]);
+      points.push([B[b][1], false, false]);
+    }
+    // Sort order: point, then start
+    points.sort((a, b) => (a[0] !== b[0] ? a[0] - b[0] : a[2] ? 1 : -1));
+    return points;
+  }
+
+  // A - B, ie, the interval with all values in A that are not in B.  Essentially
+  // a set difference operation.
+  //
+  static difference(A, B) {
+    // Corner cases
+    if (A.length === 0 || B.length === 0) {
+      return PositiveIntervals.canonicalize(A);
+    }
+
+    const cA = PositiveIntervals.canonicalize(A);
+    const cB = PositiveIntervals.canonicalize(B);
+
+    const points = PositiveIntervals._flatten(cA, cB);
+    const res = [];
+    let aDepth = 0;
+    let depth = 0;
+    let intervalStart;
+    for (let i = 0; i < points.length; i += 1) {
+      const p = points[i];
+      const delta = p[2] ? 1 : -1;
+      depth += delta;
+      if (p[1]) aDepth += delta;
+
+      if (i === points.length - 1 || p[0] !== points[i + 1][0]) {
+        if (aDepth === 1 && depth === 1) {
+          [intervalStart] = p;
+        } else if (intervalStart !== undefined) {
+          res.push([intervalStart, p[0]]);
+          intervalStart = undefined;
+        }
+      }
+    }
+    // guaranteed to be in canonical form
+    return res;
+  }
+
+  // Return interval with values belonging to A or B.  Essentially a set
+  // intersection.
+  //
+  static intersection(A, B) {
+    if (A.length === 0 || B.length === 0) {
+      return [];
+    }
+
+    const cA = PositiveIntervals.canonicalize(A);
+    const cB = PositiveIntervals.canonicalize(B);
+
+    const points = PositiveIntervals._flatten(cA, cB);
+    const res = [];
+    let depth = 0;
+    let intervalStart;
+    for (let i = 0; i < points.length; i += 1) {
+      const p = points[i];
+      depth += p[2] ? 1 : -1;
+      if (depth === 2) {
+        [intervalStart] = p;
+      } else if (intervalStart !== undefined) {
+        res.push([intervalStart, p[0]]);
+        intervalStart = undefined;
+      }
+    }
+    // guaranteed to be in canonical form
+    return res;
+  }
+}
+
+export default PositiveIntervals;