release 0.12.0 (#981)

* initial cut at backed mode

* make flask multithreading conditional on debug flag

* update X access to support backed mode

* lint

* improve help message for backed mode

* fix tests

* add MatrixProxy to normalize supported matrix types

* add FAQ entry for --backed

* remove use of matrix.T

* clean up

* add ability to disable diffexp from CLI; add hueristic to detect likely slow diffexp calculation, and warn user

* fix tests

* do not print diffexp speed warning if diffexp is disabled

* tweak wording of diffexp speed messages

* add FAQ entry on --disable-diffexp

* revise heuristic for warning about slow diffexp

* use quick tooltip delay on diffexp button

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.7.0
+current_version = 0.12.0
 
 [bumpversion:file:setup.py]
 search = version="{current_version}"
@@ -13,3 +13,7 @@ replace = version="{new_version}"
 search = "version": "{current_version}"
 replace = "version": "{new_version}"
 
+[bumpversion:file:server/__init__.py]
+search = __version__ = "{current_version}"
+replace = __version__ = "{new_version}"
+

.gitignore

@@ -27,6 +27,9 @@ server/app/web/templates/index\.html
 .ipynb_checkpoints
 *.ipynb
 
+# cefpython
+error.log
+
 # misc
 .DS_Store
 npm-debug.log

.travis.yml

@@ -1,29 +1,31 @@
 language: python
-python:
-  - "3.6"
+dist: xenial
+sudo: required
 node_js:
-  - "8"
+  - 8
 cache:
-  pip: true
+  - pip
+  - npm
 install:
   - set -eo pipefail
   - pip install flake8
-  - make build
-  - make install
+  - make pydist
+  - make install-dist
   - pip install -r server/requirements-dev.txt
-  - docker build .
 
 jobs:
   include:
-  - name: "Branch Tests"
-    script:
-      - set -eo pipefail
-      - flake8 server
-      - black --check
-      - npm run --prefix client/ build
-      - npm run --prefix client/ unit-test
-      - pytest -s server/test
+  - name: "Branch Tests 3.7"
+    python: "3.7"
+    script: ./travis-build.sh
+  - name: "Branch Tests 3.6"
+    python: "3.6"
+    script: ./travis-build.sh
+  - name: "Docker Build"
+    install: skip
+    python: "3.6"
+    script: docker build .
   - name: "Smoke Tests"
-    if: branch = master AND type = cron
+    python: "3.6"
     script:
       - npm run --prefix client/ smoke-test

CONTRIBUTING.md

@@ -0,0 +1,49 @@
+# contributing to cellxgene
+
+We warmly welcome contributions from the community! Please submit any bug reports and feature requests through [Github issues](https://github.com/chanzuckerberg/cellxgene/issues). Please submit any direct contributions by forking the repository, creating a branch, and submitting a Pull Request. It'd be great for PRs to include test cases and documentation updates where relevant, though we know the core test suite is itself still a work in progress.
+
+All code contributions and dependencies must be compatible with the project's [open-source license (MIT)](LICENSE.txt).
+
+This project adheres to the Contributor Covenant
+[code of conduct](https://github.com/chanzuckerberg/.github/tree/master/CODE_OF_CONDUCT.md).
+By participating, you are expected to uphold this code. Please report unacceptable behavior
+to opensource@chanzuckerberg.com.
+
+And finally, if you have any questions about any of this stuff, just ask! :)
+
+## developer guide
+
+This project has made a few key design choices
+
+- The front-end is built with [`regl`](https://github.com/regl-project/regl) (a webgl library), [`react`](https://reactjs.org/), [`redux`](https://redux.js.org/), [`d3`](https://github.com/d3/d3), and [`blueprint`](https://blueprintjs.com/docs/#core) to handle rendering large numbers of cells with lots of complex interactivity
+- The app is designed with a client-server model that can support a range of existing analysis packages for backend computational tasks (currently built for [scanpy](https://github.com/theislab/scanpy))
+- The client uses fast cross-filtering to handle selections and comparisons across subsets of data
+
+Depending on your background and interests, you might want to contribute to the frontend, or backend, or both!
+
+If you are interested in working on `cellxgene` development, we recommend cloning the project from Gitub. First you'll need the following installed on your machine
+
+- python 3.6+
+- node and npm (we recommend using [nvm](https://github.com/creationix/nvm) if this is your first time with node)
+
+Then clone the project
+
+```
+git clone https://github.com/chanzuckerberg/cellxgene.git
+```
+
+Build the client web assets by calling `make` from inside the `cellxgene` folder
+
+```
+make
+```
+
+Install all requirements (we recommend doing this inside a virtual environment)
+
+```
+pip install -e .
+```
+
+You can start the app while developing either by calling `cellxgene` or by calling `python -m server`. We recommend using the `--debug` flag to see more output, which you can include when reporting bugs.
+
+If you have any questions about developing or contributing, come hang out with us by joining the [CZI Science Slack](https://join-cellxgene-users.herokuapp.com/) and posting in the `#cellxgene-dev` channel.

MANIFEST.in

@@ -1,4 +1,5 @@
 recursive-include server/app/web/templates *
 recursive-include server/app/web/static *
 
-include server/requirements.txt
+include server/requirements.txt
+include server/requirements-prepare.txt

README.md

@@ -1,287 +1,89 @@
-# cellxgene
+<img src="./docs/cellxgene-logo.svg" width="300">
 
-> an interactive explorer for single-cell transcriptomics data
+_an interactive explorer for single-cell transcriptomics data_
 
-`cellxgene` is an interactive data explorer for single-cell transcriptomics datasets, such as those coming from the [Human Cell Atlas](https://humancellatlas.org). Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data, and to demonstrate general, scalable, and reusable patterns for scientific data visualization.
+[![DOI](https://zenodo.org/badge/105615409.svg)](https://zenodo.org/badge/latestdoi/105615409) [![PyPI](https://img.shields.io/pypi/v/cellxgene)](https://pypi.org/project/cellxgene/) [![PyPI - Downloads](https://img.shields.io/pypi/dm/cellxgene)](https://pypistats.org/packages/cellxgene) [![GitHub last commit](https://img.shields.io/github/last-commit/chanzuckerberg/cellxgene)](https://github.com/chanzuckerberg/cellxgene/pulse)
+
+cellxgene (pronounced "cell-by-gene") is an interactive data explorer for single-cell transcriptomics datasets, such as those coming from the [Human Cell Atlas](https://humancellatlas.org). Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data.
 
 <img src="https://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/docs/cellxgene-demo-1.gif" width="200" height="200" hspace="30"><img src="https://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/docs/cellxgene-demo-2.gif" width="200" height="200" hspace="30"><img src="https://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/docs/cellxgene-demo-3.gif" width="200" height="200" hspace="30">
 
-## getting started
+- Want to install and use cellxgene? Visit the [cellxgene docs](https://chanzuckerberg.github.io/cellxgene/).
+- Want to see where we are going? Check out [our roadmap](https://github.com/chanzuckerberg/cellxgene/blob/master/ROADMAP.md).
+- Want to contribute? See our [contributors guide](https://github.com/chanzuckerberg/cellxgene/blob/master/CONTRIBUTING.md).
 
-You'll need **python 3.6** and **Google Chrome**. (_Warning_: Python 3.7 is **not** supported at this time)
-The web UI is tested on OSX and Windows using Chrome, and the python CLI is tested on OSX and Ubuntu (via WSL/Windows). It should work on other platforms, but if you run into trouble let us know (see [help](#help-and-contact) below).
+## quick start
 
-To install run
+To install cellxgene you need Python 3.6+. We recommend [installing cellxgene into a conda or virtual environment.](https://chanzuckerberg.github.io/cellxgene/faq.html#how-do-i-create-a-python-36-environment-for-cellxgene)
 
-```
+Install the package.
+
+```bash
 pip install cellxgene
 ```
 
-To start exploring a dataset call
+Download an example [anndata](https://anndata.readthedocs.io/en/latest/) file
 
-```
-cellxgene launch dataset.h5ad --open
+```bash
+curl -O https://cellxgene-example-data.czi.technology/pbmc3k.h5ad.zip
+unzip pbmc3k.h5ad
 ```
 
-If you want an example dataset download [this file](https://github.com/chanzuckerberg/cellxgene/raw/master/example-dataset/pbmc3k.h5ad) and then call
+Launch cellxgene
 
-```
+```bash
 cellxgene launch pbmc3k.h5ad --open
 ```
 
-You should see your web browser open with the following
+To learn more about what you can do with cellxgene, see the [Getting Started](https://chanzuckerberg.github.io/cellxgene/getting-started.html) guide.
 
-<img width="450" src="https://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/docs/cellxgene-opening-screenshot.png" pad="50px">
+## get in touch
 
-**Note**: automatic opening of the browser with the `--open` flag only works on OS X, on other platforms you'll need to directly point to the provided link in your browser.
-
-There are several options available, such as:
-
-- `--layout` to specify the layout as `tsne` or `umap`
-- `--title` to show a title on the explorer
-- `--open` to automatically open the web browser after launching (OS X only)
-
-To see all options call
-
-```
-cellxgene launch --help
-```
-
-There is an additional subcommand called `cellxgene prepare` that takes an existing dataset in one of several formats and applies minimal preprocessing and reformatting so that `launch` can use it (see [the next section](##data-formatting) for more info on `prepare`).
-
-## data formatting
-
-### assumptions
-
-The `launch` command assumes that the data is stored in the `.h5ad` format from the [`anndata`](https://anndata.readthedocs.io/en/latest/index.html) library. It also assumes that certain computations have already been performed. Briefly, the `.h5ad` format wraps a two-dimensional `ndarray` and stores additional metadata as "annotations" for either observations (referred to as `obs` and `obsm`) or variables (`var` and `varm`). `cellxgene launch` makes the following assumptions about your data (we recommend loading and inspecting your data using `scanpy` to validate these assumptions)
-
-- an `obs` field has a unique identifier for every cell (you can specify which field to use with the `--obs-names` option, by default it will use the value of `data.obs_names`)
-- a `var` field has a unique identifier for every gene (you can specify which field to use with the `--var-names` option, by default it will use the value of `data.var_names`)
-- an `obsm` field contains the two-dimensional coordinates for the layout that you want to render (e.g. `X_tsne` for the `tsne` layout or `X_umap` for the `umap` layout)
-- any additional `obs` fields will be rendered as per-cell continuous or categorical metadata by the app (e.g. `louvain` cluster assignments)
-
-### prepare
-
-The `prepare` command is included to help you format your data. It uses `scanpy` under the hood. This is especially useful if you are starting with raw unanalyzed data and are unfamiliar with `scanpy`.
-
-To prepare from an existing `.h5ad` file use
-
-```
-cellxgene prepare dataset.h5ad --output=dataset-processed.h5ad
-```
-
-This will load the input data, perform PCA and nearest neighbor calculations, compute `umap` and `tsne` layouts and `louvain` cluster assignments, and save the results in a new file called `dataset-processed.h5ad` that can be loaded using `cellxgene launch`. Data can be loaded from several formats, including `.h5ad` `.loom` and a `10-Genomics-formatted` `mtx` directory. Several options are available, including running one of the preprocessing `recipes` included with `scanpy`, which include steps like cell filtering and gene selection. To learn more about the `recipes` please see the `scanpy` [documentation](https://scanpy.readthedocs.io/en/latest/api/index.html#recipes).
-
-Depending on the options chosen, `prepare` can take a long time to run (a few minutes for datasets with 10-100k cells, up to an hour or more for datasets with >100k cells). If you want `prepare` to run faster we recommend using the `sparse` option and only computing the layout for `umap`, using a call like this
-
-```
-cellxgene prepare dataset.h5ad --output=dataset-processed.h5ad --layout=umap --sparse
-```
-
-To see all options call
-
-```
-cellxgene prepare --help
-```
-
-**Note**: `cellxgene prepare` will only perform `louvain` clustering if you have the `python-igraph` and `louvain` packages installed. To make sure they are installed alongside `cellxgene` use
-
-```
-pip install cellxgene[louvain]
-```
-
-If the aforementioned optional package installation fails, you can also install these packages directly:
-
-```
-pip install python-igraph louvain>=0.6
-```
-
-## conda and virtual environments
-
-If you use conda and want to create a conda environment for `cellxgene` you can use the following commands
-
-```
-conda create --yes -n cellxgene python=3.6
-conda activate cellxgene
-pip install cellxgene
-```
-
-Or you can create a virtual environment by using
-
-```
-ENV_NAME=cellxgene
-python3.6 -m venv ${ENV_NAME}
-source ${ENV_NAME}/bin/activate
-pip install cellxgene
-```
-
-## docker
-
-We have included a dockerfile to conveniently run cellxgene from docker.
-
-1. Build the image `docker build . -t cellxgene`
-2. Run the container and mount data `docker run -v "$PWD/example-dataset/:/data/" -p 5005:5005 cellxgene launch --host 0.0.0.0 data/pbmc3k.h5ad`
-   - You will need to use --host 0.0.0.0 to have the container listen to incoming requests from the browser
-
-## FAQ
-
-<details>
-
-<summary> questions about data formatting </summary>
-  
-<hr>
-  
-> Someone sent me a directory of `10X-Genomics` data with a `mtx` file and I've never used `scanpy`, can I use `cellxgene`?
-
-Yep! This should only take a couple steps. We'll assume your data is in a folder called `data/` and you've successfully installed `cellxgene` with the `louvain` packages as described above. Just run
-
-```
-cellxgene prepare data/ --output=data-processed.h5ad --layout=umap
-```
-
-Depending on the size of the dataset, this may take some time. Once it's done, call
-
-```
-cellxgene launch data-processed.h5ad --layout=umap --open
-```
-
-And your web browser should open with an interactive view of your data.
-
-<hr>
-
-> In my `prepare` command I received the following error `Warning: louvain module is not installed, no clusters will be calculated. To fix this please install cellxgene with the optional feature louvain enabled`
-
-Louvain clustering requires additional dependencies that are somewhat complex, so we don't include them by default. For now, you need to specify that you want these packages by using
-
-```
-pip install cellxgene[louvain]
-```
-
-<hr>
-
-> I ran `prepare` and I'm getting results that look unexpected
-
-You might want to try running one of the preprocessing recipes included with `scanpy` (read more about them [here](https://scanpy.readthedocs.io/en/latest/api/index.html#recipes)). You can specify this with the `--recipe` option, such as
-
-```
-cellxgene prepare data/ --output=data-processed.h5ad --recipe=zheng17
-```
-
-It should be easy to run `prepare` then call `cellxgene launch` a few times with different settings to explore different behaviors. We may explore adding other preprocessing options in the future.
-
-<hr>
-
-> I have extra metadata that I want to add to my dataset
-
-Currently this is not supported directly, but you should be able to do this manually using `scanpy`. For example, this [notebook](https://github.com/falexwolf/fun-analyses/blob/master/tabula_muris/tabula_muris.ipynb) shows adding the contents of a `csv` file with metadata to an `anndata` object. For now, you could do this manually on your data in the same way and then save out the result before loading into `cellxgene`.
-
-<hr>
-
-> What part of the anndata objects does cellxgene pull in for visualization?
-
-- `.obs` and `.var` annotations are use to extract metadata for filtering
-- `.X` is used to display expression (histograms, scatterplot & colorscale) and to compute differential expression
-- `.obsm` is used for layout
-
-</details>
-
-<details>
-  
-<summary> questions about installing and building </summary>
-
-<hr>
-
-> I tried to `pip install cellxgene` and got a weird error about missing paths to an HDF5 library?  
-
-You probably just need to install HDF5 first. If you're on a mac, you can simply `brew install hdf5` and then try `pip install cellxgene` again.  
-
-> I tried to `pip install cellxgene` and got another weird error I don't understand  
-
-This may happen, especially as we work out bugs in our installation process! Please create a new [Github issue](https://github.com/chanzuckerberg/cellxgene/issues), explain what you did, and include all the error messages you saw. It'd also be super helpful if you call `pip freeze` and include the full output alongside your issue.  
-
-<hr>
-
-> I'm following the developer instructions and get an error about "missing files and directories” when trying to build the client
-
-This is likely because you do not have node and npm installed, we recommend using [nvm](https://github.com/creationix/nvm) if you're new to using these tools.
-
-</details>
-
-<details>
-
-<summary> questions about algorithms </summary>
-
-<hr>
-
-> How are you computing and sorting differential expression results?
-
-Currently we use a [Welch's _t_-test](https://en.wikipedia.org/wiki/Welch%27s_t-test) implementation including the same variance overestimation correction as used in `scanpy`. We sort the `tscore` to identify the top N genes, and then filter to remove any that fall below a cutoff log fold change value, which can help remove spurious test results. The default threshold is `0.01` and can be changed using the option `--diffexp-lfc-cutoff`. We can explore adding support for other test types in the future.
-
-</details>
-
-## developer guide
-
-This project has made a few key design choices
-
-- The front-end is built with [`regl`](https://github.com/regl-project/regl) (a webgl library), [`react`](https://reactjs.org/), [`redux`](https://redux.js.org/), [`d3`](https://github.com/d3/d3), and [`blueprint`](https://blueprintjs.com/docs/#core) to handle rendering large numbers of cells with lots of complex interactivity
-- The app is designed with a client-server model that can support a range of existing analysis packages for backend computational tasks (currently built for [scanpy](https://github.com/theislab/scanpy))
-- The client uses fast cross-filtering to handle selections and comparisons across subsets of data
-
-Depending on your background and interests, you might want to contribute to the frontend, or backend, or both!
-
-If you are interested in working on `cellxgene` development, we recommend cloning the project from Gitub. First you'll need the following installed on your machine
-
-- python 3.6
-- node and npm (we recommend using [nvm](https://github.com/creationix/nvm) if this is your first time with node)
-
-Then clone the project
-
-```
-git clone https://github.com/chanzuckerberg/cellxgene.git
-```
-
-Build the client web assets by calling this from inside the `cellxgene` folder
-
-```
-./bin/build-client
-```
-
-Install all requirements (we recommend doing this inside a virtual environment)
-
-```
-pip install -e .
-```
-
-You can start the app while developing either by calling `cellxgene` or by calling `python -m server`. We recommend using the `--debug` flag to see more output, which you can include when reporting bugs.
-
-If you have any questions about developing or contributing, come hang out with us by joining the [CZI Science Slack](https://join-cellxgene-users.herokuapp.com/) and posting in the `#cellxgene-dev` channel.
-
-## development roadmap
-
-`cellxgene` is still very much in development, and we've love to include the community as we plan new features to work on. We are thinking about working on the following features over the next 3-12 months. If you are interested in updates, want to give feedback, want to contribute, or have ideas about other features we should work on, please [contact us](#help-and-contact)
-
-- **Visualizaling spatial metadata** Image-based transcriptomics methods also generate large cell by gene matrices, alongside rich metadata about spatial location; we would like to render this information in `cellxgene`
-- **Visualizing trajectories** Trajectory analyses infer progression along some ordering or pseudotime; we would like `cellxgene` to render the results of these analyses when they have been performed
-- **Deploy to web** Many projects release public data browser websites alongside their publicatons; we would like to make it easy for anyone to deploy `cellxgene` to a custom URL with their own dataset that they own and operate
-- **HCA Integration** The [Human Cell Atlas](https://humancellatlas.org) is generating a large corpus of single-cell expression data and will make it available through the Data Coordination Platform; we would like `cellxgene` to be one of several different portals for browsing these data
+Have questions, suggestions, or comments? You can come hang out with us by joining the [CZI Science Slack](https://join-cellxgene-users.herokuapp.com/) and posting in the `#cellxgene-users` channel. Have feature requests or bugs? Please submit these as [Github issues](https://github.com/chanzuckerberg/cellxgene/issues). We'd love to hear from you!
 
 ## contributing
 
-We warmly welcome contributions from the community! Please submit any bug reports and feature requests through [Github issues](https://github.com/chanzuckerberg/cellxgene/issues). Please submit any direct contributions by forking the repository, creating a branch, and submitting a Pull Request. It'd be great for PRs to include test cases and documentation updates where relevant, though we know the core test suite is itself still a work in progress. And all code contributions and dependencies must be compatible with the project's open-source license (MIT). If you have any questions about this stuff, just ask!
+We warmly welcome contributions from the community! Please see our [contributing guide](https://github.com/chanzuckerberg/cellxgene/blob/master/CONTRIBUTING.md) and don't hesitate to open an issue or send a pull request to improve cellxgene.
 
-## inspiration and collaboration
+This project adheres to the Contributor Covenant [code of conduct](https://github.com/chanzuckerberg/.github/blob/master/CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to opensource@chanzuckerberg.com.
+
+## core team
+
+The current core team:
+
+- Colin Megill, frontend & product design
+- Bruce Martin, software engineer
+- Sidney Bell, computational biologist
+- Lia Prins, designer
+- Severiano Badajoz, software engineer
+
+We would also like to gratefully acknowledge contributions from past core team members:
+
+- Charlotte Weaver, software engineer
+
+## where we are going
+
+Our goal is to enable teams of computational and experimental
+biologists to collaboratively gain insight into their single-cell RNA-seq data.
+
+There are 4 key features we plan to implement in the near term.
+
+- Click install and launch
+- Manual annotation workflows
+- Toggle embeddings
+- Gene information
+
+For more detail on these features and where we are going, see [our roadmap](https://github.com/chanzuckerberg/cellxgene/blob/master/ROADMAP.md).
+
+## inspiration
 
 We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browswer](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [Gene Pattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data.
 
 We were inspired by Mike Bostock and the [crossfilter](https://github.com/crossfilter) team for the design of our filtering implementation.
 
-We have been working closely with the [`scanpy`](https://github.com/theislab/scanpy) team to integrate with their awesome analysis tools. Special thanks to Alex Wolf, Fabian Theis, and the rest of the team for their help during development and for providing an example dataset.
+We have been working closely with the [scanpy](https://github.com/theislab/scanpy) team to integrate with their awesome analysis tools. Special thanks to Alex Wolf, Fabian Theis, and the rest of the team for their help during development and for providing an example dataset.
 
-We are eager to explore integrations with other computational backends such as [`Seurat`](https://github.com/satijalab/seurat) or [`Bioconductor`](https://github.com/Bioconductor)
-
-## help and contact
-
-Have questions, suggestions, or comments? You can come hang out with us by joining the [CZI Science Slack](https://join-cellxgene-users.herokuapp.com/) and posting in the `#cellxgene-users` channel. As mentioned above, please submit any feature requests or bugs as [Github issues](https://github.com/chanzuckerberg/cellxgene/issues). We'd love to hear from you!
+We are eager to explore integrations with other computational backends such as [Seurat](https://github.com/satijalab/seurat) or [Bioconductor](https://github.com/Bioconductor)
 
 ## reuse
 

ROADMAP.md

@@ -0,0 +1,46 @@
+# cellxgene roadmap
+
+cellxgene makes it easier for biologists to collaboratively explore and understand their single-cell RNA-seq data. 
+In the near term, we are focused on continuing to enable fast, interactive exploration of single-cell data, supporting collaborative workflows in single-cell analysis, and improving user support. 
+If you have questions or feedback about this roadmap, please submit an issue on GitHub.
+Please note: this roadmap is subject to change.
+
+Last updated: June 25, 2019
+
+## Fast, interactive exploration of single-cell data
+
+### Exposing Relationships Between Metadata and Data
+Biologists need to understand how variables (stored in metadata) are associated with one another and how they relate to changes in gene expression. 
+Building upon visualization features that reveal categorical metadata relationships (cluster occupancy) and gene expression relationships (scatterplot), we plan to add exploratory visualization components that enable investigation of relationships between metadata and gene expression. 
+See [issue #616](https://github.com/chanzuckerberg/cellxgene/issues/616) for more details.
+
+### Contextualizing Genes
+While exploring a transcriptomics dataset, scientists need to understand the biological context of genes.
+This context may be provided by user-defined gene metadata or publicly available gene databases. 
+We plan to support augmenting gene names with additional information that is useful to biologists. 
+See [issue #96](https://github.com/chanzuckerberg/cellxgene/issues/96) for more detail.
+
+## Support collaborative workflows in single-cell analysis
+
+### Manual Annotations
+cellxgene offers exploratory visualizations that are critical for manual annotation workflows, especially in collaborative environments. 
+We plan to support manually annotating cells with labels (i.e., cell type or QC flags), and their easy export for downstream analysis. 
+See [issue #524](https://github.com/chanzuckerberg/cellxgene/issues/524) for more details.
+
+### Simple Click to Launch
+
+Many biologists prefer not to interact with the command line and need an OS-native experience when using cellxgene. 
+We plan to implement a point-and-click installation and launch experience so that users can easily load data into cellxgene. 
+See [issue #687](https://github.com/chanzuckerberg/cellxgene/issues/687) for details.
+
+### Python API
+For computational biologists, saving h5ad files then loading them into cellxgene is a point of friction.  
+We plan to support importing cellxgene as a Python package so that users can launch cellxgene directly from an interactive environment (such as Jupyter, IPython, or Spyder), and pass data to and from the cellxgene UI. 
+
+## Improving user support
+
+### Improved documentation
+cellxgene has some specific expectations about how data is stored. 
+We want to ensure that new users can get started easily and learn how to use cellxgene with their own data. 
+We plan to improve documentation on getting started, installation, data, and contributing. 
+See [issue #533](https://github.com/chanzuckerberg/cellxgene/issues/533) for more details.

app.json

@@ -0,0 +1,21 @@
+{
+  "name": "cellxgene",
+  "description": "An interactive explorer for single-cell transcriptomics data",
+  "repository": "https://github.com/chanzuckerberg/cellxgene/",
+  "logo": "https://cellxgene-example-data.czi.technology/favicon.png",
+  "keywords": [
+    "scientific",
+    "visualization",
+    "scrna-seq",
+    "transcriptomics",
+    "dataviz"
+  ],
+  "stack": "container",
+  "env": {
+    "DATASET": {
+      "description": "Link to dataset",
+      "value": "https://cellxgene-example-data.czi.technology/pbmc3k.h5ad",
+      "required": "true"
+    }
+  }
+}

cellxgene-osx.spec

@@ -0,0 +1,41 @@
+# -*- mode: python ; coding: utf-8 -*-
+
+block_cipher = None
+
+
+a = Analysis(['server/gui/main.py'],
+             pathex=['/Users/charlotteweaver/Documents/Git/cellxgene'],
+             binaries=[('/System/Library/Frameworks/Tk.framework/Tk', 'tk'), ('/System/Library/Frameworks/Tcl.framework/Tcl', 'tcl')],
+             datas=[('server/app/web/templates/', 'server/app/web/templates/'), ('server/app/web/static/', 'server/app/web/static/')],
+             hiddenimports=['sklearn', 'sklearn.utils._cython_blas', 'sklearn.neighbors.typedefs', 'sklearn.neighbors.quad_tree', 'sklearn.tree', 'sklearn.tree._utils'],
+             hookspath=['server/gui/'],
+             runtime_hooks=[],
+             excludes=[],
+             win_no_prefer_redirects=False,
+             win_private_assemblies=False,
+             cipher=block_cipher,
+             noarchive=False)
+pyz = PYZ(a.pure, a.zipped_data,
+             cipher=block_cipher)
+exe = EXE(pyz,
+          a.scripts,
+          [],
+          exclude_binaries=True,
+          name='cellxgene',
+          debug=False,
+          bootloader_ignore_signals=False,
+          strip=False,
+          upx=True,
+          console=False , icon='server/gui/images/cxg_icons.icns')
+coll = COLLECT(exe,
+               a.binaries,
+               a.zipfiles,
+               a.datas,
+               strip=False,
+               upx=True,
+               upx_exclude=[],
+               name='cellxgene')
+app = BUNDLE(coll,
+             name='cellxgene.app',
+             icon='server/gui/images/cxg_icons.icns',
+             bundle_identifier=None)

cellxgene-windows.spec

@@ -0,0 +1,36 @@
+# -*- mode: python -*-
+
+block_cipher = None
+
+
+a = Analysis(['server\\gui\\main.py'],
+             pathex=['C:\\Users\\Charlotte\\Documents\\git\\cellxgene'],
+             binaries=[],
+             datas=[('server/app/web/templates/', 'server/app/web/templates'), ('server/app/web/static/', 'server/app/web/static')],
+             hiddenimports=[],
+             hookspath=['server/gui/'],
+             runtime_hooks=[],
+             excludes=[],
+             win_no_prefer_redirects=False,
+             win_private_assemblies=False,
+             cipher=block_cipher,
+             noarchive=False)
+pyz = PYZ(a.pure, a.zipped_data,
+             cipher=block_cipher)
+exe = EXE(pyz,
+          a.scripts,
+          [],
+          exclude_binaries=True,
+          name='cellxgene',
+          debug=False,
+          bootloader_ignore_signals=False,
+          strip=False,
+          upx=True,
+          console=False , icon='server\\gui\\images\\icon.ico')
+coll = COLLECT(exe,
+               a.binaries,
+               a.zipfiles,
+               a.datas,
+               strip=False,
+               upx=True,
+               name='cellxgene')

client/__tests__/e2e/config.js

@@ -0,0 +1,6 @@
+export const jest_env = process.env.JEST_ENV || "dev";
+export const appPort = process.env.JEST_CXG_PORT || 3000;
+export const appUrlBase = `http://localhost:${appPort}`;
+export const DEV = jest_env === "dev";
+export const DEBUG = jest_env === "debug";
+export const DATASET = "pbmc3k";

client/__tests__/e2e/data.js

@@ -0,0 +1,128 @@
+export const datasets = {
+  pbmc3k: {
+    title: "pbmc3k",
+    dataframe: {
+      nObs: "2638",
+      nVar: "1838",
+      type: "float32"
+    },
+    categorical: {
+      louvain: {
+        "B cells": "342",
+        "CD14+ Monocytes": "480",
+        "CD4 T cells": "1144",
+        "CD8 T cells": "316",
+        "Dendritic cells": "37",
+        "FCGR3A+ Monocytes": "150",
+        Megakaryocytes: "15",
+        "NK cells": "154"
+      }
+    },
+    continuous: {
+      n_genes: "int32",
+      percent_mito: "float32",
+      n_counts: "float32"
+    },
+    cellsets: {
+      lasso: [
+        {
+          "coordinates-as-percent": { x1: 0.05, y1: 0.25, x2: 0.15, y2: 0.35 },
+          count: "88"
+        }
+      ],
+      categorical: [
+        {
+          metadata: "louvain",
+          values: ["B cells", "Megakaryocytes"],
+          count: "357"
+        }
+      ],
+      continuous: [
+        {
+          metadata: "n_genes",
+          "coordinates-as-percent": { x1: 0.25, y1: 0.5, x2: 0.55, y2: 0.5 },
+          count: "1537"
+        }
+      ]
+    },
+
+    diffexp: {
+      cellset1: [
+        { kind: "categorical", metadata: "louvain", values: ["B cells"] }
+      ],
+      cellset2: [
+        {
+          kind: "categorical",
+          metadata: "louvain",
+          values: ["CD4 T cells", "NK cells"]
+        }
+      ],
+      "gene-results": [
+        "HLA-DRB1",
+        "HLA-DPB1",
+        "CD79A",
+        "HLA-DPA1",
+        "HLA-DQA1",
+        "CD79B",
+        "HLA-DQB1",
+        "MS4A1",
+        "IL32",
+        "CD37"
+      ]
+    },
+
+    genes: {
+      bulkadd: ["S100A8", "FCGR3A", "LGALS2", "GSTP1"],
+      search: "ACD"
+    },
+    subset: {
+      cellset1: [
+        {
+          kind: "categorical",
+          metadata: "louvain",
+          values: ["B cells", "Megakaryocytes"]
+        }
+      ],
+      count: "357",
+      categorical: {
+        louvain: {
+          "B cells": "342",
+          "CD14+ Monocytes": "0",
+          "CD4 T cells": "0",
+          "CD8 T cells": "0",
+          "Dendritic cells": "0",
+          "FCGR3A+ Monocytes": "0",
+          Megakaryocytes: "15",
+          "NK cells": "0"
+        }
+      },
+      lasso: {
+        "coordinates-as-percent": { x1: 0.45, y1: 0.05, x2: 0.65, y2: 0.15 },
+        count: "39"
+      }
+    },
+    scatter: {
+      genes: { x: "S100A8", y: "FCGR3A" }
+    },
+    pan: {
+      "coordinates-as-percent": { x1: 0.75, y1: 0.75, x2: 0.35, y2: 0.35 }
+    },
+    features: {
+      panzoom: {
+        lasso: {
+          "coordinates-as-percent": { x1: 0.3, y1: 0.3, x2: 0.5, y2: 0.5 },
+          count: "24"
+        }
+      }
+    },
+    clip: {
+      min: "30",
+      max: "70",
+      metadata: "n_genes",
+      gene: "S100A8",
+      "coordinates-as-percent": { x1: 0.25, y1: 0.5, x2: 0.55, y2: 0.5 },
+      count: "392",
+      "gene-cell-count": "421"
+    }
+  }
+};

client/__tests__/e2e/e2e.test.js

@@ -1,185 +1,343 @@
+/*
+Smoke test suite that will be run in Travis CI
+
+Tests included in this file are expected to be relatively stable and test core features
+ */
 import puppeteer from "puppeteer";
+import { appUrlBase, DEBUG, DEV, DATASET } from "./config";
+import { puppeteerUtils, cellxgeneActions } from "./puppeteerUtils";
+import { datasets } from "./data";
 
-const jest_env = process.env.JEST_ENV || "dev";
-const appPort = process.env.JEST_CXG_PORT || 3000;
-const appUrlBase = `http://localhost:${appPort}`;
-const DEV = jest_env === "dev";
-
-let browser;
-let page;
+let browser, page, utils, cxgActions, spy;
 const browserViewport = { width: 1280, height: 960 };
+let data = datasets[DATASET];
+
+if (DEBUG) jest.setTimeout(100000);
+if (DEV) jest.setTimeout(10000);
 
 beforeAll(async () => {
-  const browser_params = DEV
+  const browserParams = DEV
+    ? { headless: false, slowMo: 5 }
+    : DEBUG
     ? { headless: false, slowMo: 100, devtools: true }
     : {};
-  browser = await puppeteer.launch(browser_params);
+  browser = await puppeteer.launch(browserParams);
   page = await browser.newPage();
-  page.setViewport(browserViewport);
-  if (DEV) page.on("console", msg => console.log("PAGE LOG:", msg.text()));
+  await page.setViewport(browserViewport);
+  if (DEV || DEBUG) {
+    page.on("console", async msg => {
+      // If there is a console.error but an error is not thrown, this will ensure the test fails
+      if (msg.type() === "error") {
+        const errorMsgText = await Promise.all(
+          // TODO can we do this without internal properties?
+          msg.args().map(arg => arg._remoteObject.description)
+        );
+        throw new Error(`Console error: ${errorMsgText}`);
+      }
+      console.log(`PAGE LOG: ${msg.text()}`);
+    });
+  }
+  page.on("pageerror", err => {
+    throw new Error(`Console error: ${err}`);
+  });
+  utils = puppeteerUtils(page);
+  cxgActions = cellxgeneActions(page);
+});
+
+beforeEach(async () => {
+  await page.goto(appUrlBase);
 });
 
 afterAll(() => {
-  if (!DEV) {
+  if (!DEBUG) {
     browser.close();
   }
 });
 
-const getOneElementInnerHTML = async function(selector) {
-  let text = await page.$eval(selector, el => el.innerHTML);
-  return text;
-};
-
-const drag = async function(el_box, start, end, lasso = false) {
-  const x1 = el_box.content[0].x + start.x;
-  const x2 = el_box.content[0].x + end.x;
-  const y1 = el_box.content[0].y + start.y;
-  const y2 = el_box.content[0].y + end.y;
-  await page.mouse.move(x1, y1);
-  await page.mouse.down();
-  if (lasso) {
-    await page.mouse.move(x2, y1);
-    await page.mouse.move(x2, y2);
-    await page.mouse.move(x1, y2);
-    await page.mouse.move(x1, y1);
-  } else {
-    await page.mouse.move(x2, y2);
-  }
-  await page.mouse.up();
-};
-
 describe("did launch", () => {
   test("page launched", async () => {
-    await page.goto(appUrlBase);
-    let el = await getOneElementInnerHTML("[data-testid='header']");
-    expect(el).toBe("cellxgene: pbmc3k");
+    let el = await utils.getOneElementInnerHTML("[data-testid='header']");
+    expect(el).toBe(data.title);
   });
 });
 
-describe("search for genes", () => {
-  test("search for known gene and add to metadata", async () => {
-    await page.goto(appUrlBase);
-    await page.waitForSelector("[ data-testid='gene-search']");
-    // blueprint's  typeahead is treating typing weird, clicking & waiting first solves this
-    await page.click("[data-testid='gene-search']");
-    await page.waitFor(200);
-    await page.type("[data-testid='gene-search']", "ACD");
-    await page.keyboard.press("Enter");
-    await page.waitForSelector("[data-testid='histogram-ACD']");
+describe("metadata loads", () => {
+  test("categories and values from dataset appear", async () => {
+    for (const label in data.categorical) {
+      await utils.waitByID(`category-${label}`);
+      const categoryName = await utils.getOneElementInnerText(
+        `[data-testid="category-${label}"]`
+      );
+      expect(categoryName).toMatch(label);
+      await utils.clickOn(`category-expand-${label}`);
+      const categories = await cxgActions.getAllCategoriesAndCounts(label);
+      expect(Object.keys(categories)).toMatchObject(
+        Object.keys(data.categorical[label])
+      );
+      expect(Object.values(categories)).toMatchObject(
+        Object.values(data.categorical[label])
+      );
+    }
+  });
+
+  test("continuous data appears", async () => {
+    for (const label in data.continuous) {
+      await utils.waitByID(`histogram-${label}`);
+    }
   });
 });
 
-describe("select cells and diffexp", () => {
-  test("selects cells from layout and adds to cell set 1", async () => {
-    await page.goto(appUrlBase);
-    const layout = await page.waitForSelector("[data-testid='layout']");
-    const size = await layout.boxModel();
-    const cellset1 = {
-      start: {
-        x: Math.floor(size.width * 0.25),
-        y: Math.floor(size.height * 0.25)
-      },
-      end: {
-        x: Math.floor(size.width * 0.35),
-        y: Math.floor(size.height * 0.35)
-      }
-    };
-    await drag(size, cellset1.start, cellset1.end, true);
-    await page.click("[data-testid='cellset-button-1");
-    let button = await getOneElementInnerHTML("[data-testid='cellset-button-1");
-    expect(button).toMatch(/26 cells/);
+describe("cell selection", () => {
+  test("selects all cells cellset 1", async () => {
+    const cellCount = await cxgActions.cellSet(1);
+    expect(cellCount).toBe(data.dataframe.nObs);
   });
 
-  test("selects cells from layout and adds to cell set 2", async () => {
-    await page.goto(appUrlBase);
-    const layout = await page.waitForSelector("[data-testid='layout']");
-    const size = await layout.boxModel();
-    const cellset2 = {
-      start: {
-        x: Math.floor(size.width * 0.45),
-        y: Math.floor(size.height * 0.45)
-      },
-      end: {
-        x: Math.floor(size.width * 0.55),
-        y: Math.floor(size.height * 0.55)
-      }
-    };
-    await drag(size, cellset2.start, cellset2.end, true);
-    await page.click("[data-testid='cellset-button-2");
-    let button = await getOneElementInnerHTML("[data-testid='cellset-button-2");
-    expect(button).toMatch(/49 cells/);
+  test("selects all cells cellset 2", async () => {
+    const cellCount = await cxgActions.cellSet(2);
+    expect(cellCount).toBe(data.dataframe.nObs);
   });
 
-  test("selects cells, saves them and performs diffexp", async () => {
-    await page.goto(appUrlBase);
-    const layout = await page.waitForSelector("[data-testid='layout']");
-    const size = await layout.boxModel();
-    const cellset1 = {
-      start: {
-        x: Math.floor(size.width * 0.25),
-        y: Math.floor(size.height * 0.25)
-      },
-      end: {
-        x: Math.floor(size.width * 0.35),
-        y: Math.floor(size.height * 0.35)
-      }
-    };
-    await drag(size, cellset1.start, cellset1.end, true);
-    await page.click("[data-testid='cellset-button-1");
-    const cellset2 = {
-      start: {
-        x: Math.floor(size.width * 0.45),
-        y: Math.floor(size.height * 0.45)
-      },
-      end: {
-        x: Math.floor(size.width * 0.55),
-        y: Math.floor(size.height * 0.55)
-      }
-    };
-    await drag(size, cellset2.start, cellset2.end, true);
-    await page.click("[data-testid='cellset-button-2");
-    await page.click("[data-testid='diffexp-button");
-    await page.waitForSelector("[data-testclass='histogram-diffexp']");
-    const diffexps = await page.$$eval(
-      "[data-testclass='histogram-diffexp']",
-      divs => {
-        return divs.map(div =>
-          div.id.substring("histogram-".length, div.id.length)
+  test("selects cells via lasso", async () => {
+    for (const cellset of data.cellsets.lasso) {
+      const cellset1 = await cxgActions.calcDragCoordinates(
+        "layout-graph",
+        cellset["coordinates-as-percent"]
+      );
+      await cxgActions.drag("layout-graph", cellset1.start, cellset1.end, true);
+      const cellCount = await cxgActions.cellSet(1);
+      expect(cellCount).toBe(cellset.count);
+    }
+  });
+
+  test("selects cells via categorical", async () => {
+    for (const cellset of data.cellsets.categorical) {
+      await utils.clickOn(`category-expand-${cellset.metadata}`);
+      await utils.clickOn(`category-select-${cellset.metadata}`);
+      for (const val of cellset.values) {
+        await utils.clickOn(
+          `categorical-value-select-${cellset.metadata}-${val}`
         );
       }
-    );
-    expect(diffexps).toMatchObject([
-      "HLA-DPA1",
-      "HLA-DQA1",
-      "HLA-DRB1",
-      "HLA-DMA",
-      "CST3",
-      "HLA-DPB1",
-      "HLA-DQB1",
-      "LGALS2",
-      "FCER1A",
-      "LTB"
-    ]);
+      const cellCount = await cxgActions.cellSet(1);
+      expect(cellCount).toBe(cellset.count);
+    }
+  });
+
+  test("selects cells via continuous", async () => {
+    for (const cellset of data.cellsets.continuous) {
+      const histId = `histogram-${cellset.metadata}-plot-brush`;
+      const coords = await cxgActions.calcDragCoordinates(
+        histId,
+        cellset["coordinates-as-percent"]
+      );
+      await cxgActions.drag(histId, coords.start, coords.end);
+      const cellCount = await cxgActions.cellSet(1);
+      expect(cellCount).toBe(cellset.count);
+    }
   });
 });
 
-describe("brushable histogram", () => {
-  test("can brush historgram", async () => {
-    await page.goto(appUrlBase);
-    const hist = await page.waitForSelector(
-      "[data-testid='histogram_n_genes_svg-brush'] > .overlay"
+describe("gene entry", () => {
+  test("search for single gene", async () => {
+    // blueprint's  typeahead is treating typing weird, clicking & waiting first solves this
+    await utils.typeInto("gene-search", data.genes.search);
+    await page.keyboard.press("Enter");
+    await page.waitForSelector(
+      `[data-testid='histogram-${data.genes.search}']`
     );
-    const hist_size = await hist.boxModel();
-    const draghist = {
-      start: {
-        x: Math.floor(hist_size.width * 0.25),
-        y: Math.floor(hist_size.height * 0.5)
-      },
-      end: {
-        x: Math.floor(hist_size.width * 0.55),
-        y: Math.floor(hist_size.height * 0.5)
-      }
-    };
-    await drag(hist_size, draghist.start, draghist.end);
+  });
+
+  test("bulk add genes", async () => {
+    await cxgActions.reset();
+    const testGenes = data.genes.bulkadd;
+    await utils.clickOn("section-bulk-add");
+    await utils.typeInto("input-bulk-add", testGenes.join(","));
+    await page.keyboard.press("Enter");
+
+    const allHistograms = await cxgActions.getAllHistograms(
+      "histogram-user-gene",
+      testGenes
+    );
+    expect(allHistograms).toEqual(expect.arrayContaining(testGenes));
+    expect(allHistograms.length).toEqual(testGenes.length);
+  });
+});
+
+describe("diffexp", () => {
+  test("selects cells, saves them and performs diffexp", async () => {
+    for (const select of data.diffexp.cellset1) {
+      if (select.kind === "categorical") {
+        await cxgActions.selectCategory(select.metadata, select.values, true);
+      }
+    }
+    await cxgActions.cellSet(1);
+    for (const select of data.diffexp.cellset2) {
+      if (select.kind === "categorical") {
+        await cxgActions.selectCategory(select.metadata, select.values, true);
+      }
+    }
+    await cxgActions.cellSet(2);
+    await utils.clickOn("diffexp-button");
+    const allHistograms = await cxgActions.getAllHistograms(
+      "histogram-diffexp",
+      data.diffexp["gene-results"]
+    );
+    expect(allHistograms).toEqual(
+      expect.arrayContaining(data.diffexp["gene-results"])
+    );
+    expect(allHistograms.length).toEqual(data.diffexp["gene-results"].length);
+  });
+});
+
+describe("subset/reset", () => {
+  test("subset - cell count matches", async () => {
+    for (const select of data.subset.cellset1) {
+      if (select.kind === "categorical") {
+        await cxgActions.selectCategory(select.metadata, select.values, true);
+      }
+    }
+    await utils.clickOn("subset-button");
+    for (const label in data.subset.categorical) {
+      const categories = await cxgActions.getAllCategoriesAndCounts(label);
+      expect(Object.keys(categories)).toMatchObject(
+        Object.keys(data.subset.categorical[label])
+      );
+      expect(Object.values(categories)).toMatchObject(
+        Object.values(data.subset.categorical[label])
+      );
+    }
+  });
+
+  test("reset after subset", async () => {
+    for (const select of data.subset.cellset1) {
+      if (select.kind === "categorical") {
+        await cxgActions.selectCategory(select.metadata, select.values, true);
+      }
+    }
+    await utils.clickOn("subset-button");
+    for (const label in data.subset.categorical) {
+      const categories = await cxgActions.getAllCategoriesAndCounts(label);
+      expect(Object.keys(categories)).toMatchObject(
+        Object.keys(data.subset.categorical[label])
+      );
+      expect(Object.values(categories)).toMatchObject(
+        Object.values(data.subset.categorical[label])
+      );
+    }
+    await cxgActions.reset();
+    for (const label in data.categorical) {
+      await utils.waitByID(`category-${label}`);
+      const categoryName = await utils.getOneElementInnerText(
+        `[data-testid="category-${label}"]`
+      );
+      expect(categoryName).toMatch(label);
+      const categories = await cxgActions.getAllCategoriesAndCounts(label);
+      expect(Object.keys(categories)).toMatchObject(
+        Object.keys(data.categorical[label])
+      );
+      expect(Object.values(categories)).toMatchObject(
+        Object.values(data.categorical[label])
+      );
+    }
+  });
+
+  test("lasso after subset", async () => {
+    for (const select of data.subset.cellset1) {
+      if (select.kind === "categorical") {
+        await cxgActions.selectCategory(select.metadata, select.values, true);
+      }
+    }
+    await utils.clickOn("subset-button");
+    const lassoSelection = await cxgActions.calcDragCoordinates(
+      "layout-graph",
+      data.subset.lasso["coordinates-as-percent"]
+    );
+    await cxgActions.drag(
+      "layout-graph",
+      lassoSelection.start,
+      lassoSelection.end,
+      true
+    );
+    const cellCount = await cxgActions.cellSet(1);
+    expect(cellCount).toBe(data.subset.lasso.count);
+  });
+});
+
+describe("scatter plot", () => {
+  test("scatter plot appears", async () => {
+    await cxgActions.reset();
+    const testGenes = data.scatter.genes;
+    await utils.clickOn("section-bulk-add");
+    await utils.typeInto("input-bulk-add", Object.values(testGenes).join(","));
+    await page.keyboard.press("Enter");
+    await utils.clickOn(`plot-x-${data.scatter.genes.x}`);
+    await utils.clickOn(`plot-y-${data.scatter.genes.y}`);
+    await utils.waitByID("scatterplot");
+  });
+});
+
+describe("clipping", () => {
+  test("clip continuous", async () => {
+    await cxgActions.clip(data.clip.min, data.clip.max);
+    const histId = `histogram-${data.clip.metadata}-plot-brush`;
+    const coords = await cxgActions.calcDragCoordinates(
+      histId,
+      data.clip["coordinates-as-percent"]
+    );
+    await cxgActions.drag(histId, coords.start, coords.end);
+    const cellCount = await cxgActions.cellSet(1);
+    expect(cellCount).toBe(data.clip.count);
+  });
+
+  test("clip gene", async () => {
+    await utils.typeInto("gene-search", data.clip.gene);
+    await page.keyboard.press("Enter");
+    await page.waitForSelector(`[data-testid='histogram-${data.clip.gene}']`);
+    await cxgActions.clip(data.clip.min, data.clip.max);
+    const histId = `histogram-${data.clip.gene}-plot-brush`;
+    const coords = await cxgActions.calcDragCoordinates(
+      histId,
+      data.clip["coordinates-as-percent"]
+    );
+    await cxgActions.drag(histId, coords.start, coords.end);
+    const cellCount = await cxgActions.cellSet(1);
+    expect(cellCount).toBe(data.clip["gene-cell-count"]);
+  });
+});
+
+// interact with UI elements just that they do not break
+describe("ui elements don't error", () => {
+  test("color by", async () => {
+    for (const label in data.categorical) {
+      await utils.clickOn(`colorby-${label}`);
+    }
+    for (const label in data.continuous) {
+      await utils.clickOn(`colorby-${label}`);
+    }
+  });
+
+  test("color by for gene", async () => {
+    await utils.typeInto("gene-search", data.genes.search);
+    await page.keyboard.press("Enter");
+    await page.waitForSelector(
+      `[data-testid='histogram-${data.genes.search}']`
+    );
+    await utils.clickOn(`colorby-${data.genes.search}`);
+  });
+
+  test("pan and zoom", async () => {
+    await utils.clickOn("mode-pan-zoom");
+    const panCoords = await cxgActions.calcDragCoordinates(
+      "layout-graph",
+      data.pan["coordinates-as-percent"]
+    );
+    await cxgActions.drag(
+      "layout-graph",
+      panCoords.start,
+      panCoords.end,
+      false
+    );
+    await page.evaluate(`window.scrollBy(0, 1000);`);
   });
 });

client/__tests__/e2e/e2eJestConfig.json

@@ -0,0 +1,10 @@
+{
+  "preset": "jest-puppeteer",
+  "testMatch": [
+    "**/__tests__/**/?(*.)(spec|test).js?(x)"
+  ],
+  "testURL": "http://localhost/",
+  "setupFiles": [
+    "../setupMissingGlobals.js"
+  ]
+}

client/__tests__/e2e/feature.test.js

@@ -0,0 +1,118 @@
+/*
+NOT run in Travis CI
+
+UX tests using puppeteer to be run locally.
+
+To run locally, ensure you are running the client is running on port 3000.
+Then run jest --verbose false --config __tests__/e2e/e2eJestConfig.json feature.
+ */
+
+import puppeteer from "puppeteer";
+import { appUrlBase, DEBUG, DEV, DATASET } from "./config";
+import { puppeteerUtils, cellxgeneActions } from "./puppeteerUtils";
+import { datasets } from "./data";
+
+let browser, page, utils, cxgActions, spy;
+const browserViewport = { width: 1280, height: 960 };
+let data = datasets[DATASET].features;
+
+if (DEBUG) jest.setTimeout(100000);
+if (DEV) jest.setTimeout(10000);
+
+beforeAll(async () => {
+  const browserParams = DEV
+    ? { headless: false, slowMo: 5 }
+    : DEBUG
+    ? { headless: false, slowMo: 100, devtools: true }
+    : {};
+  browser = await puppeteer.launch(browserParams);
+  page = await browser.newPage();
+  await page.setViewport(browserViewport);
+  if (DEV || DEBUG) {
+    page.on("console", msg => console.log(`PAGE LOG: ${msg.text()}`));
+  }
+  page.on("pageerror", err => {
+    throw new Error(`Console error: ${err}`);
+  });
+  utils = puppeteerUtils(page);
+  cxgActions = cellxgeneActions(page);
+});
+
+beforeEach(async () => {
+  await page.goto(appUrlBase);
+});
+
+afterAll(() => {
+  if (!DEBUG) {
+    browser.close();
+  }
+});
+
+describe("zoom interaction", async () => {
+  // Skip this test since UI is to hide lasso path when switching modes
+  test.skip("lasso visible after switching modes to pan/zoom", async () => {
+    const lassoSelection = await cxgActions.calcDragCoordinates(
+      "layout-graph",
+      data.panzoom.lasso["coordinates-as-percent"]
+    );
+    await cxgActions.drag(
+      "layout-graph",
+      lassoSelection.start,
+      lassoSelection.end,
+      true
+    );
+    await utils.waitByID("lasso-element", { visible: true });
+    await utils.clickOn("mode-pan-zoom");
+    await utils.waitByID("lasso-element", { visible: true });
+  });
+
+  test("pan zoom mode resets lasso selection", async () => {
+    const lassoSelection = await cxgActions.calcDragCoordinates(
+      "layout-graph",
+      data.panzoom.lasso["coordinates-as-percent"]
+    );
+    await cxgActions.drag(
+      "layout-graph",
+      lassoSelection.start,
+      lassoSelection.end,
+      true
+    );
+    await utils.waitByID("lasso-element", { visible: true });
+    const initialCount = await cxgActions.cellSet(1);
+    expect(initialCount).toBe(data.panzoom.lasso.count);
+    await utils.clickOn("mode-pan-zoom");
+    await utils.clickOn("mode-lasso");
+    const modeSwitchCount = await cxgActions.cellSet(1);
+    expect(modeSwitchCount).toBe(initialCount);
+  });
+
+  test("lasso moves after pan", async () => {
+    const lassoSelection = await cxgActions.calcDragCoordinates(
+      "layout-graph",
+      data.panzoom.lasso["coordinates-as-percent"]
+    );
+    await cxgActions.drag(
+      "layout-graph",
+      lassoSelection.start,
+      lassoSelection.end,
+      true
+    );
+    await utils.waitByID("lasso-element", { visible: true });
+    const initialCount = await cxgActions.cellSet(1);
+    expect(initialCount).toBe(data.panzoom.lasso.count);
+    await utils.clickOn("mode-pan-zoom");
+    const panCoords = await cxgActions.calcDragCoordinates(
+      "layout-graph",
+      data.panzoom.lasso["coordinates-as-percent"]
+    );
+    await cxgActions.drag(
+      "layout-graph",
+      panCoords.start,
+      panCoords.end,
+      false
+    );
+    await utils.clickOn("mode-lasso");
+    const panCount = await cxgActions.cellSet(2);
+    expect(panCount).toBe(initialCount);
+  });
+});

client/__tests__/e2e/puppeteerUtils.js

@@ -0,0 +1,211 @@
+export const puppeteerUtils = puppeteerPage => ({
+  async waitByID(testid, props = {}) {
+    return await puppeteerPage.waitForSelector(
+      `[data-testid='${testid}']`,
+      props
+    );
+  },
+
+  async waitByClass(testclass, props = {}) {
+    return await puppeteerPage.waitForSelector(
+      `[data-testclass='${testclass}']`,
+      props
+    );
+  },
+
+  async waitForAllByIds(testids, props = {}) {
+    await Promise.all(
+      testids.map(testid =>
+        puppeteerPage.waitForSelector(`[data-testid='${testid}']`)
+      )
+    );
+  },
+
+  async getAllByClass(testclass, props = {}) {
+    const elements = await puppeteerPage.$$eval(
+      `[data-testclass=${testclass}]`,
+      els => {
+        return els.map(el => {
+          return el.dataset.testid;
+        });
+      }
+    );
+    return elements;
+  },
+
+  async typeInto(testid, text) {
+    // only works for text without special characters
+    await this.waitByID(testid);
+    const selector = `[data-testid='${testid}']`;
+    // type ahead can be annoying if you don't pause before you type
+    await puppeteerPage.click(selector);
+    await puppeteerPage.waitFor(200);
+    await puppeteerPage.type(selector, text);
+  },
+
+  async clearInputAndTypeInto(testid, text) {
+    await this.waitByID(testid);
+    const selector = `[data-testid='${testid}']`;
+    // only works for text without special characters
+    // type ahead can be annoying if you don't pause before you type
+    await puppeteerPage.click(selector);
+    await puppeteerPage.waitFor(200);
+    // select all
+
+    await puppeteerPage.click(selector, { clickCount: 3 });
+    await puppeteerPage.keyboard.type("Backspace");
+    await puppeteerPage.type(selector, text);
+  },
+
+  async clickOn(testid) {
+    await this.waitByID(testid);
+    await puppeteerPage.click(`[data-testid='${testid}']`);
+    await puppeteerPage.waitFor(50);
+  },
+
+  async getOneElementInnerHTML(selector) {
+    await puppeteerPage.waitForSelector(selector);
+    let text = await puppeteerPage.$eval(selector, el => el.innerHTML);
+    return text;
+  },
+
+  async getOneElementInnerText(selector) {
+    await puppeteerPage.waitForSelector(selector);
+    let text = await puppeteerPage.$eval(selector, el => el.innerText);
+    return text;
+  }
+});
+
+export const cellxgeneActions = puppeteerPage => ({
+  async drag(testid, start, end, lasso = false) {
+    const layout = await puppeteerUtils(puppeteerPage).waitByID(testid);
+    const elBox = await layout.boxModel();
+    const x1 = elBox.content[0].x + start.x;
+    const x2 = elBox.content[0].x + end.x;
+    const y1 = elBox.content[0].y + start.y;
+    const y2 = elBox.content[0].y + end.y;
+    await puppeteerPage.mouse.move(x1, y1);
+    await puppeteerPage.mouse.down();
+    if (lasso) {
+      await puppeteerPage.mouse.move(x2, y1);
+      await puppeteerPage.mouse.move(x2, y2);
+      await puppeteerPage.mouse.move(x1, y2);
+      await puppeteerPage.mouse.move(x1, y1);
+    } else {
+      await puppeteerPage.mouse.move(x2, y2);
+    }
+    await puppeteerPage.mouse.up();
+  },
+
+  async getAllHistograms(testclass, testids) {
+    const histTestIds = testids.map(tid => `histogram-${tid}`);
+    // these load asynchronously, so we need to wait for each histogram individually
+    await puppeteerUtils(puppeteerPage).waitForAllByIds(histTestIds);
+    const allHistograms = await puppeteerUtils(puppeteerPage).getAllByClass(
+      testclass
+    );
+    return allHistograms.map(hist =>
+      hist.substr("histogram_".length, hist.length)
+    );
+  },
+
+  async getAllCategoriesAndCounts(category) {
+    await puppeteerUtils(puppeteerPage).waitByClass("categorical-row");
+    const categories = await puppeteerPage.$$eval(
+      `[data-testid="category-${category}"] [data-testclass='categorical-row']`,
+      els => {
+        let result = {};
+        els.forEach(el => {
+          const cat = el.querySelector("[data-testclass='categorical-value']")
+            .innerText;
+          const count = el.querySelector(
+            "[data-testclass='categorical-value-count']"
+          ).innerText;
+          result[cat] = count;
+        });
+        return result;
+      }
+    );
+    return categories;
+  },
+
+  async cellSet(num) {
+    await puppeteerUtils(puppeteerPage).clickOn(`cellset-button-${num}`);
+    return await puppeteerUtils(puppeteerPage).getOneElementInnerText(
+      `[data-testid='cellset-count-${num}']`
+    );
+  },
+
+  async resetCategory(category) {
+    const checkboxId = `category-select-${category}`;
+    await puppeteerUtils(puppeteerPage).waitByID(checkboxId);
+    const checkedPseudoclass = await puppeteerPage.$eval(
+      `[data-testid='${checkboxId}']`,
+      el => {
+        return el.matches(":checked");
+      }
+    );
+    if (!checkedPseudoclass) {
+      await puppeteerUtils(puppeteerPage).clickOn(checkboxId);
+    }
+    try {
+      const categoryRow = await puppeteerUtils(puppeteerPage).waitByID(
+        `category-expand-${category}`
+      );
+      const isExpanded = await categoryRow.$(
+        "[data-testclass='category-expand-is-expanded']"
+      );
+      if (isExpanded) {
+        await puppeteerUtils(puppeteerPage).clickOn(
+          `category-expand-${category}`
+        );
+      }
+    } catch {}
+  },
+
+  async calcDragCoordinates(testid, coordinateAsPercent) {
+    const el = await puppeteerUtils(puppeteerPage).waitByID(testid);
+    const size = await el.boxModel();
+    const coords = {
+      start: {
+        x: Math.floor(size.width * coordinateAsPercent.x1),
+        y: Math.floor(size.height * coordinateAsPercent.y1)
+      },
+      end: {
+        x: Math.floor(size.width * coordinateAsPercent.x2),
+        y: Math.floor(size.height * coordinateAsPercent.y2)
+      }
+    };
+    return coords;
+  },
+
+  async selectCategory(category, values, reset = true) {
+    if (reset) await this.resetCategory(category);
+    await puppeteerUtils(puppeteerPage).clickOn(`category-expand-${category}`);
+    await puppeteerUtils(puppeteerPage).clickOn(`category-select-${category}`);
+    for (const val of values) {
+      await puppeteerUtils(puppeteerPage).clickOn(
+        `categorical-value-select-${category}-${val}`
+      );
+    }
+  },
+
+  async reset() {
+    await puppeteerUtils(puppeteerPage).clickOn("reset");
+    // loading state never actually happens, reset is too fast
+    await page.waitFor(200);
+  },
+
+  async clip(min = 0, max = 100) {
+    await puppeteerUtils(puppeteerPage).clickOn("visualization-settings");
+    await puppeteerUtils(puppeteerPage).clearInputAndTypeInto(
+      "clip-min-input",
+      min
+    );
+    await puppeteerUtils(puppeteerPage).clearInputAndTypeInto(
+      "clip-max-input",
+      max
+    );
+    await puppeteerUtils(puppeteerPage).clickOn("clip-commit");
+  }
+});

client/__tests__/reducers/cascade.test.js

@@ -0,0 +1,48 @@
+import cascadeReducers from "../../src/reducers/cascade";
+
+describe("create", () => {
+  test("from Array", () => {
+    expect(cascadeReducers([["foo", () => 0]])).toBeInstanceOf(Function);
+  });
+
+  test("from Map", () => {
+    expect(cascadeReducers(new Map([["foo", () => 0]]))).toBeInstanceOf(
+      Function
+    );
+  });
+});
+
+describe("cascade", () => {
+  test("expected arguments provided & cascade ordering", () => {
+    const topLevelState = {};
+    const topLevelAction = { type: "test" };
+
+    const reducer = cascadeReducers([
+      [
+        "foo",
+        (currentState, action, nextSharedState, prevSharedState) => {
+          expect(currentState).toBeUndefined();
+          expect(action).toEqual(topLevelAction);
+          expect(nextSharedState).toStrictEqual({});
+          expect(prevSharedState).toBe(topLevelState);
+          return 0;
+        }
+      ],
+      [
+        "bar",
+        (currentState, action, nextSharedState, prevSharedState) => {
+          expect(currentState).toBeUndefined();
+          expect(action).toEqual(topLevelAction);
+          expect(nextSharedState).toStrictEqual({ foo: 0 });
+          expect(prevSharedState).toBe(topLevelState);
+          return 99;
+        }
+      ]
+    ]);
+
+    const nextState = reducer(topLevelState, topLevelAction);
+    expect(nextState).toStrictEqual({ foo: 0, bar: 99 });
+    expect(topLevelState).toStrictEqual({});
+    expect(topLevelAction).toStrictEqual({ type: "test" });
+  });
+});

client/__tests__/reducers/undoable.test.js

@@ -0,0 +1,87 @@
+import undoable from "../../src/reducers/undoable";
+
+describe("create", () => {
+  test("no keys", () => {
+    expect(() => undoable(() => {})).toThrow();
+    expect(() => undoable(() => {}, null)).toThrow();
+    expect(() => undoable(() => {}, [])).toThrow();
+    expect(() => undoable(() => {}, [], {})).toThrow();
+  });
+
+  test("simple", () => {
+    expect(undoable(() => {}, ["foo"])).toBeInstanceOf(Function);
+    expect(undoable(() => {}, ["foo"], {})).toBeInstanceOf(Function);
+  });
+
+  test("handles undefined initial state", () => {
+    expect(
+      undoable(() => {}, ["a"])(undefined, { type: "test" })
+    ).toMatchObject({});
+  });
+});
+
+describe("undo", () => {
+  test("expected state modifications", () => {
+    const initialState = { a: 0, b: 1000 };
+    const reducer = state => {
+      return { a: state.a + 1, b: state.b + 1 };
+    };
+    const undoableReducer = undoable(reducer, ["a"]);
+
+    const s1 = undoableReducer(initialState, { type: "test" });
+    expect(s1).toMatchObject({ a: 1, b: 1001 });
+
+    // test that only specified keys are undone
+    const s2 = undoableReducer(s1, { type: "@@undoable/undo" });
+    expect(s2).toMatchObject({ a: 0, b: 1001 });
+
+    // test backstop when no more history
+    const s3 = undoableReducer(s2, { type: "@@undoable/undo" });
+    expect(s3).toMatchObject({ a: 0, b: 1001 });
+  });
+});
+
+describe("redo", () => {
+  const initialState = { a: 0, b: 1000 };
+  const reducer = state => {
+    return { a: state.a + 1, b: state.b + 1 };
+  };
+  let UR;
+
+  beforeEach(() => {
+    UR = undoable(reducer, ["a"]);
+  });
+
+  test("expected state modifications", () => {
+    const s1 = UR(initialState, { type: "test" });
+    expect(s1).toMatchObject({ a: 1, b: 1001 });
+
+    // verify undo->redo reverts state.
+    const s2 = UR(UR(s1, { type: "@@undoable/undo" }), {
+      type: "@@undoable/redo"
+    });
+    expect(s2).toMatchObject({ a: 1, b: 1001 });
+
+    // verify backstop when no redo future
+    const s3 = UR(s2, { type: "@@undoable/redo" });
+    expect(s3).toMatchObject({ a: 1, b: 1001 });
+  });
+
+  test("history cleared", () => {
+    // verify future cleared upon a normal state transition
+    const s1 = UR(initialState, { type: "test" });
+    expect(s1).toMatchObject({ a: 1, b: 1001 });
+    const s2 = UR(s1, { type: "@@undoable/undo" });
+    expect(s2).toMatchObject({ a: 0, b: 1001 });
+    const s3 = UR(s2, { type: "test" });
+    expect(s3).toMatchObject({ a: 1, b: 1002 });
+    const s4 = UR(s3, { type: "@@undoable/redo" });
+    expect(s4).toMatchObject({ a: 1, b: 1002 });
+  });
+});
+
+/*
+TODO:
+- historyLimit is enforced
+- action filters
+*/

client/__tests__/util/dataframe/dataframe.test.js

@@ -452,6 +452,59 @@ describe("dataframe factories", () => {
     });
   });
 
+  describe("withColsFrom", () => {
+    test("error conditions", () => {
+      /*
+      make sure we catch common errors:
+      - duplicate column names
+      - dimensionality difference
+      */
+      const dfA = new Dataframe.Dataframe(
+        [2, 3],
+        [["red", "blue"], [true, false], [1, 0]],
+        null,
+        new Dataframe.KeyIndex(["colors", "bools", "numbers"])
+      );
+
+      /* different dimensionality should throw error */
+      const dfB = new Dataframe.Dataframe(
+        [3, 1],
+        [["red", "blue", "green"]],
+        null,
+        new Dataframe.KeyIndex(["colorsA"])
+      );
+      expect(() => dfA.withColsFrom(dfB)).toThrow(RangeError);
+
+      /* duplicate labels should throw an error */
+      expect(() => dfA.withColsFrom(dfA)).toThrow(Error);
+    });
+
+    test("simple", () => {
+      /* simple test that it works as expected in common case */
+      const dfA = new Dataframe.Dataframe(
+        [2, 1],
+        [["red", "blue"]],
+        null,
+        new Dataframe.KeyIndex(["colors"])
+      );
+      const dfB = new Dataframe.Dataframe(
+        [2, 1],
+        [[true, false]],
+        null,
+        new Dataframe.KeyIndex(["bools"])
+      );
+
+      const dfC = dfA.withColsFrom(dfB);
+      expect(dfC).toBeDefined();
+      expect(dfC.dims).toEqual([2, 2]);
+      expect(dfC.colIndex.keys()).toEqual(["colors", "bools"]);
+      expect(dfC.rowIndex).toEqual(dfA.rowIndex);
+      expect(dfC.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(dfC.col("colors").asArray()).toEqual(["red", "blue"]);
+      expect(dfC.col("bools").asArray()).toEqual([true, false]);
+    });
+  });
+
   describe("dropCol", () => {
     test("KeyIndex", () => {
       const df = new Dataframe.Dataframe(
@@ -532,6 +585,68 @@ describe("dataframe factories", () => {
       expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
     });
   });
+
+  describe("mapColumns", () => {
+    test("identity", () => {
+      const dfA = Dataframe.Dataframe.create(
+        [3, 3],
+        [
+          new Array(3).fill(0),
+          new Int16Array(3).fill(99),
+          new Float64Array(3).fill(1.1)
+        ]
+      );
+      const dfB = dfA.mapColumns((col, idx) => {
+        expect(dfA.icol(idx).asArray()).toBe(col);
+        return col;
+      });
+      expect(dfA).not.toBe(dfB);
+      expect(dfA.dims).toEqual(dfB.dims);
+      for (let c = 0; c < dfA.dims[1]; c += 1) {
+        expect(dfA.icol(c).asArray()).toBe(dfB.icol(c).asArray());
+      }
+    });
+
+    test("transform", () => {
+      const dfA = Dataframe.Dataframe.create(
+        [3, 3],
+        [new Array(3).fill(0), new Array(3).fill(0), new Array(3).fill(0)]
+      );
+      const dfB = dfA.mapColumns(() => {
+        return new Array(3).fill(1);
+      });
+      expect(dfA).not.toBe(dfB);
+      expect(dfB.iat(0, 0)).toEqual(1);
+      expect(dfB.iat(0, 1)).toEqual(1);
+      expect(dfB.iat(0, 2)).toEqual(1);
+    });
+
+    test("columns", () => {
+      const df = Dataframe.Dataframe.create(
+        [3, 3],
+        [new Array(3).fill(0), new Array(3).fill(0), new Array(3).fill(0)]
+      );
+
+      expect(df).toBeDefined();
+      expect(df.columns()).toHaveLength(3);
+      expect(df.columns()[0]).toEqual(df.icol(0));
+      expect(df.columns()[2]).toEqual(df.icol(2));
+    });
+
+    test("renameCol", () => {
+      const dfA = new Dataframe.Dataframe(
+        [2, 2],
+        [[true, false], [1, 0]],
+        null,
+        new Dataframe.KeyIndex(["A", "B"])
+      );
+      const dfB = dfA.renameCol("B", "C");
+      expect(dfA.colIndex.keys()).toEqual(["A", "B"]);
+      expect(dfB.colIndex.keys()).toEqual(["A", "C"]);
+      expect(dfA.dims).toMatchObject(dfB.dims);
+      expect(dfA.columns()).toMatchObject(dfB.columns());
+    });
+  });
 });
 
 describe("dataframe col", () => {

client/__tests__/util/dataframe/histogram.test.js

@@ -0,0 +1,93 @@
+import * as Dataframe from "../../../src/util/dataframe";
+
+describe("Dataframe column histogram", () => {
+  test("categorical by categorical", () => {
+    const df = new Dataframe.Dataframe(
+      [3, 3],
+      [["n1", "n2", "n3"], ["c1", "c2", "c3"], new Int32Array([0, 1, 2])],
+      null,
+      new Dataframe.KeyIndex(["name", "cat", "value"])
+    );
+
+    const h1 = df.col("cat").histogram(df.col("name"));
+    expect(h1).toMatchObject(
+      new Map([
+        ["n1", new Map([["c1", 1]])],
+        ["n2", new Map([["c2", 1]])],
+        ["n3", new Map([["c3", 1]])]
+      ])
+    );
+    // memoized?
+    expect(df.col("cat").histogram(df.col("name"))).toMatchObject(h1);
+  });
+
+  test("continuous by categorical", () => {
+    const df = new Dataframe.Dataframe(
+      [3, 3],
+      [["n1", "n2", "n3"], ["c1", "c2", "c3"], new Int32Array([0, 1, 2])],
+      null,
+      new Dataframe.KeyIndex(["name", "cat", "value"])
+    );
+
+    const h1 = df.col("value").histogram(3, [0, 2], df.col("name"));
+    expect(h1).toMatchObject(
+      new Map([["n1", [1, 0, 0]], ["n2", [0, 1, 0]], ["n3", [0, 0, 1]]])
+    );
+    // memoized?
+    expect(df.col("value").histogram(3, [0, 2], df.col("name"))).toMatchObject(
+      h1
+    );
+  });
+
+  test("categorical", () => {
+    const df = new Dataframe.Dataframe(
+      [3, 3],
+      [["n1", "n2", "n3"], ["c1", "c2", "c3"], new Int32Array([0, 1, 2])],
+      null,
+      new Dataframe.KeyIndex(["name", "cat", "value"])
+    );
+
+    const h1 = df.col("cat").histogram();
+    expect(h1).toMatchObject(new Map([["c1", 1], ["c2", 1], ["c3", 1]]));
+    // memoized?
+    expect(df.col("value").histogram(3, [0, 2])).toMatchObject(h1);
+  });
+
+  test("continuous", () => {
+    const df = new Dataframe.Dataframe(
+      [3, 3],
+      [["n1", "n2", "n3"], ["c1", "c2", "c3"], new Int32Array([0, 1, 2])],
+      null,
+      new Dataframe.KeyIndex(["name", "cat", "value"])
+    );
+
+    const h1 = df.col("value").histogram(3, [0, 2]);
+    expect(h1).toMatchObject([1, 1, 1]);
+    // memoized?
+    expect(df.col("value").histogram(3, [0, 2])).toMatchObject(h1);
+  });
+
+  test("continuous thesholds correct", () => {
+    const vals = [0, 1, 9, 10, 11, 20, 99, 100];
+    const df = new Dataframe.Dataframe(
+      [8, 2],
+      [new Int32Array(vals), new Float32Array(vals)]
+    );
+
+    expect(df.col(0).histogram(5, [0, 100])).toEqual([5, 1, 0, 0, 2]);
+    expect(df.col(1).histogram(5, [0, 100])).toEqual([5, 1, 0, 0, 2]);
+    expect(df.col(0).histogram(2, [0, 10])).toEqual([2, 2]);
+    expect(df.col(0).histogram(10, [0, 100])).toEqual([
+      3,
+      2,
+      1,
+      0,
+      0,
+      0,
+      0,
+      0,
+      0,
+      2
+    ]);
+  });
+});

client/__tests__/util/quantile.test.js

@@ -0,0 +1,29 @@
+import quantile from "../../src/util/quantile";
+
+describe("quantile", () => {
+	test("single q", () => {
+		const arr = new Float32Array([9, 3, 5, 6, 0]);
+		expect(quantile([1.0], arr)).toMatchObject([9]);
+		expect(quantile([0.9], arr)).toMatchObject([9]);
+		expect(quantile([0.8], arr)).toMatchObject([9]);
+		expect(quantile([0.7], arr)).toMatchObject([6]);
+		expect(quantile([0.6], arr)).toMatchObject([6]);
+		expect(quantile([0.5], arr)).toMatchObject([5]);
+		expect(quantile([0.4], arr)).toMatchObject([5]);
+		expect(quantile([0.3], arr)).toMatchObject([3]);
+		expect(quantile([0.2], arr)).toMatchObject([3]);
+		expect(quantile([0.1], arr)).toMatchObject([0]);
+		expect(quantile([0], arr)).toMatchObject([0]);
+	});
+
+	test("multi q", () => {
+		const arr = new Float32Array([9, 3, 5, 6, 0]);
+		expect(quantile([0, 0.25, 0.5, 0.75, 1.0], arr)).toMatchObject([
+			0,
+			3,
+			5,
+			6,
+			9
+		]);
+	});
+});

client/__tests__/util/range.test.js

@@ -0,0 +1,42 @@
+import { range, rangeFill } from "../../src/util/range";
+
+describe("range", () => {
+	test("no defaults", () => {
+		expect(range(0, 3, 1)).toMatchObject([0, 1, 2]);
+	});
+
+	test("range(stop)", () => {
+		expect(range(3)).toMatchObject([0, 1, 2]);
+		expect(range(0)).toMatchObject([]);
+		expect(range(1)).toMatchObject([0]);
+	});
+
+	test("range(start,stop)", () => {
+		expect(range(0, 0)).toMatchObject([]);
+		expect(range(0, 2)).toMatchObject([0, 1]);
+		expect(range(4, 8)).toMatchObject([4, 5, 6, 7]);
+	});
+
+	test("range(start, stop, step", () => {
+		expect(range(4, 0, -1)).toMatchObject([4, 3, 2, 1]);
+		expect(range(0, 4, 2)).toMatchObject([0, 2]);
+	});
+});
+
+describe("rangefill", () => {
+	test("rangeFill(arr)", () => {
+		expect(rangeFill(new Int32Array(3))).toMatchObject(
+			new Int32Array([0, 1, 2])
+		);
+	});
+	test("rangeFill(arr, start)", () => {
+		expect(rangeFill(new Int32Array(2), 1)).toMatchObject(
+			new Int32Array([1, 2])
+		);
+	});
+	test("rangeFill(arr, start, step)", () => {
+		expect(rangeFill(new Int32Array(3), 2, -1)).toMatchObject(
+			new Int32Array([2, 1, 0])
+		);
+	});
+});

client/__tests__/util/stateManager/fbs.test.js

@@ -0,0 +1,34 @@
+/*
+test FBS encode/decode API
+*/
+import { Dataframe, KeyIndex } from "../../../src/util/dataframe";
+import {
+	decodeMatrixFBS,
+	encodeMatrixFBS
+} from "../../../src/util/stateManager/matrix";
+
+describe("encode/decode", () => {
+	test("round trip", () => {
+		const columns = [
+			["red", "green", "blue"],
+			new Int32Array(3).fill(0),
+			new Uint32Array(3).fill(1),
+			new Float32Array(3).fill(2)
+		];
+
+		const dfNoColIdx = new Dataframe([3, 4], columns);
+		const dfA = decodeMatrixFBS(encodeMatrixFBS(dfNoColIdx));
+		expect([dfA.nRows, dfA.nCols]).toEqual(dfNoColIdx.dims);
+		expect(dfA.colIdx).toBeNull();
+		expect(dfA.rowIdx).toBeNull();
+		expect(dfA.columns).toEqual(columns);
+
+		const colIndex = new KeyIndex(["a", "b", "c", "d"]);
+		const dfWithColIdx = new Dataframe([3, 4], columns, null, colIndex);
+		const dfB = decodeMatrixFBS(encodeMatrixFBS(dfWithColIdx));
+		expect([dfB.nRows, dfB.nCols]).toEqual(dfWithColIdx.dims);
+		expect(dfB.colIdx).toEqual(colIndex.keys());
+		expect(dfB.rowIdx).toBeNull();
+		expect(dfB.columns).toEqual(columns);
+	});
+});

client/__tests__/util/stateManager/sampleResponses.js

@@ -34,28 +34,38 @@ const aSchemaResponse = {
       type: "float32"
     },
     annotations: {
-      obs: [
-        { name: "name", type: "string" },
-        { name: "field1", type: "int32" },
-        { name: "field2", type: "float32" },
-        { name: "field3", type: "boolean" },
-        {
-          name: "field4",
-          type: "categorical",
-          categories: field4Categories
-        }
-      ],
-      var: [
-        { name: "name", type: "string" },
-        { name: "fieldA", type: "int32" },
-        { name: "fieldB", type: "float32" },
-        { name: "fieldC", type: "boolean" },
-        {
-          name: "fieldD",
-          type: "categorical",
-          categories: fieldDCategories
-        }
-      ]
+      obs: {
+        index: "name",
+        columns: [
+          { name: "name", type: "string" },
+          { name: "field1", type: "int32" },
+          { name: "field2", type: "float32" },
+          { name: "field3", type: "boolean" },
+          {
+            name: "field4",
+            type: "categorical",
+            categories: field4Categories
+          }
+        ]
+      },
+      var: {
+        index: "name",
+        columns: [
+          { name: "name", type: "string" },
+          { name: "fieldA", type: "int32" },
+          { name: "fieldB", type: "float32" },
+          { name: "fieldC", type: "boolean" },
+          {
+            name: "fieldD",
+            type: "categorical",
+            categories: fieldDCategories
+          }
+        ]
+      }
+    },
+    layout: {
+      obs: [{ name: "umap", type: "float32", dims: ["umap_0", "umap_1"] }],
+      var: []
     }
   }
 };
@@ -100,6 +110,14 @@ function encodeTypedArray(builder, uType, uData) {
 }
 
 function encodeMatrix(columns, colIndex = undefined) {
+  /*
+  IMPORTANT: this is not a general purpose encoder.  in particular,
+  it doesn't correctly handle all column index types, nor does it 
+  handle all column typedarray types.
+
+  encodeMatrixFBS in matrix.py is more general.  This is used only
+  as a testing santity check (alt implementation).
+  */
   const utf8Encoder = new TextEncoder("utf-8");
   const builder = new flatbuffers.Builder(1024);
   const cols = _.map(columns, carr => {
@@ -162,29 +180,7 @@ const aLayoutFBSResponse = (() => {
     new Float32Array(nObs).fill(Math.random()),
     new Float32Array(nObs).fill(Math.random())
   ];
-  const builder = new flatbuffers.Builder(1024);
-
-  const cols = _.map(coords, carr => {
-    const cdv = NetEncoding.Float32Array.createDataVector(builder, carr);
-    NetEncoding.Float32Array.startFloat32Array(builder);
-    NetEncoding.Float32Array.addData(builder, cdv);
-    const floatArr = NetEncoding.Float32Array.endFloat32Array(builder);
-
-    NetEncoding.Column.startColumn(builder);
-    NetEncoding.Column.addUType(builder, NetEncoding.TypedArray.Float32Array);
-    NetEncoding.Column.addU(builder, floatArr);
-    return NetEncoding.Column.endColumn(builder);
-  });
-
-  const columns = NetEncoding.Matrix.createColumnsVector(builder, cols);
-
-  NetEncoding.Matrix.startMatrix(builder);
-  NetEncoding.Matrix.addNRows(builder, nObs);
-  NetEncoding.Matrix.addNCols(builder, coords.length);
-  NetEncoding.Matrix.addColumns(builder, columns);
-  const matrix = NetEncoding.Matrix.endMatrix(builder);
-  builder.finish(matrix);
-  return builder.asUint8Array();
+  return encodeMatrix(coords, ["umap_0", "umap_1"]);
 })();
 
 const aDataObsResponse = {

client/__tests__/util/stateManager/universe.test.js

@@ -53,13 +53,15 @@ describe("createUniverseFromResponse", () => {
 
     expect(universe.obsAnnotations.dims).toEqual([
       nObs,
-      REST.schema.schema.annotations.obs.length
+      REST.schema.schema.annotations.obs.columns.length
     ]);
     expect(universe.obsLayout.dims).toEqual([nObs, 2]);
-    expect(universe.obsLayout.colIndex.keys()).toEqual(["X", "Y"]);
+    expect(universe.obsLayout.colIndex.keys()).toEqual(
+      universe.schema.layout.obs[0].dims
+    );
     expect(universe.varAnnotations.dims).toEqual([
       nVar,
-      REST.schema.schema.annotations.var.length
+      REST.schema.schema.annotations.var.columns.length
     ]);
     expect(universe.varData.isEmpty()).toBeTruthy();
   });

client/__tests__/util/stateManager/world.test.js

@@ -3,6 +3,7 @@ import * as Universe from "../../../src/util/stateManager/universe";
 import * as World from "../../../src/util/stateManager/world";
 import * as Dataframe from "../../../src/util/dataframe";
 import Crossfilter from "../../../src/util/typedCrossfilter";
+import { DimTypes } from "../../../src/util/typedCrossfilter/crossfilter";
 import * as REST from "./sampleResponses";
 import {
   obsAnnoDimensionName,
@@ -17,35 +18,36 @@ const defaultBigBang = () => {
   /* create unverse, world, crossfilter and dimensionMap */
   /* create universe */
   const universe = Universe.createUniverseFromResponse(
-    REST.config,
-    REST.schema,
-    REST.annotationsObs,
-    REST.annotationsVar,
-    REST.layoutObs
+    _.cloneDeep(REST.config),
+    _.cloneDeep(REST.schema),
+    _.cloneDeep(REST.annotationsObs),
+    _.cloneDeep(REST.annotationsVar),
+    _.cloneDeep(REST.layoutObs)
   );
   /* create world */
   const world = World.createWorldFromEntireUniverse(universe);
   /* create crossfilter */
-  const crossfilter = Crossfilter(world.obsAnnotations);
-  /* create dimension map */
-  const dimensionMap = World.createObsDimensionMap(crossfilter, world);
+  const crossfilter = World.createObsDimensions(
+    new Crossfilter(world.obsAnnotations),
+    world,
+    REST.schema.schema.layout.obs[0].dims
+  );
 
   return {
     universe,
     world,
-    crossfilter,
-    dimensionMap
+    crossfilter
   };
 };
 
 describe("createWorldFromEntireUniverse", () => {
   test("create from REST sample", () => {
     const universe = Universe.createUniverseFromResponse(
-      REST.config,
-      REST.schema,
-      REST.annotationsObs,
-      REST.annotationsVar,
-      REST.layoutObs
+      _.cloneDeep(REST.config),
+      _.cloneDeep(REST.schema),
+      _.cloneDeep(REST.annotationsObs),
+      _.cloneDeep(REST.annotationsVar),
+      _.cloneDeep(REST.layoutObs)
     );
     expect(universe).toBeDefined();
 
@@ -57,10 +59,15 @@ describe("createWorldFromEntireUniverse", () => {
         nObs: universe.nObs,
         nVar: universe.nVar,
         schema: universe.schema,
-        obsAnnotations: universe.obsAnnotations,
-        varAnnotations: universe.varAnnotations,
-        obsLayout: universe.obsLayout,
-        varData: expect.any(Dataframe.Dataframe)
+        obsAnnotations: expect.any(Dataframe.Dataframe),
+        varAnnotations: expect.any(Dataframe.Dataframe),
+        obsLayout: expect.any(Dataframe.Dataframe),
+        varData: expect.any(Dataframe.Dataframe),
+        clipQuantiles: { min: 0, max: 1 },
+        unclipped: {
+          obsAnnotations: expect.any(Dataframe.Dataframe),
+          varData: expect.any(Dataframe.Dataframe)
+        }
       })
     );
   });
@@ -71,22 +78,25 @@ describe("createWorldFromCurrentSelection", () => {
     const {
       universe,
       world: originalWorld,
-      crossfilter,
-      dimensionMap
+      crossfilter: originalCrossfilter
     } = defaultBigBang();
 
     /* mock a selection */
-    dimensionMap[obsAnnoDimensionName("field1")].filterRange([0, 5]);
-    dimensionMap[obsAnnoDimensionName("field3")].filterExact(false);
+    const crossfilter = originalCrossfilter
+      .select(obsAnnoDimensionName("field1"), { mode: "range", lo: 0, hi: 5 })
+      .select(obsAnnoDimensionName("field3"), {
+        mode: "exact",
+        values: [false]
+      });
 
     /* create the world from the selection */
-    const world = World.createWorldFromCurrentSelection(
+    const world = World.createWorldBySelection(
       universe,
       originalWorld,
       crossfilter
     );
     expect(world).toBeDefined();
-    expect(world.nObs).toEqual(crossfilter.countFiltered());
+    expect(world.nObs).toEqual(crossfilter.countSelected());
 
     /*
     calculate expected values and match against result
@@ -108,10 +118,15 @@ describe("createWorldFromCurrentSelection", () => {
         nObs: matchingIndices.length,
         nVar: universe.nVar,
         schema: universe.schema,
+        clipQuantiles: { min: 0, max: 1 },
         obsAnnotations: expect.any(Dataframe.Dataframe),
-        varAnnotations: universe.varAnnotations,
+        varAnnotations: expect.any(Dataframe.Dataframe),
         obsLayout: expect.any(Dataframe.Dataframe),
-        varData: expect.any(Dataframe.Dataframe)
+        varData: expect.any(Dataframe.Dataframe),
+        unclipped: {
+          obsAnnotations: expect.any(Dataframe.Dataframe),
+          varData: expect.any(Dataframe.Dataframe)
+        }
       })
     );
 
@@ -124,7 +139,9 @@ describe("createWorldFromCurrentSelection", () => {
     expect(world.obsLayout.rowIndex.keys()).toEqual(
       new Int32Array(matchingIndices)
     );
-    expect(world.obsLayout.colIndex.keys()).toEqual(["X", "Y"]);
+    expect(world.obsLayout.colIndex.keys()).toEqual(
+      world.schema.layout.obs[0].dims
+    );
   });
 });
 
@@ -136,43 +153,36 @@ describe("createObsDimensionMap", () => {
     - check that dimension typing is sane
     */
 
-    const { dimensionMap } = defaultBigBang();
+    const { crossfilter } = defaultBigBang();
     const annotationNames = _.map(
-      REST.schema.schema.annotations.obs,
+      REST.schema.schema.annotations.obs.columns,
       c => c.name
     );
-    const schemaByObsName = _.keyBy(REST.schema.schema.annotations.obs, "name");
-    expect(dimensionMap).toBeDefined();
+    const obsIndexColName = REST.schema.schema.annotations.obs.index;
+    const schemaByObsName = _.keyBy(
+      REST.schema.schema.annotations.obs.columns,
+      "name"
+    );
+    expect(crossfilter).toBeDefined();
     annotationNames.forEach(name => {
-      const dim = dimensionMap[obsAnnoDimensionName(name)];
-      if (name === "name") {
+      const dim = crossfilter.dimensions[obsAnnoDimensionName(name)];
+      if (name === obsIndexColName) {
         expect(dim).toBeUndefined();
       } else {
         const { type } = schemaByObsName[name];
         if (type === "string" || type === "boolean" || type === "categorical") {
-          expect(dim).toBeInstanceOf(Crossfilter.EnumDimension);
+          expect(dim.dim).toBeInstanceOf(DimTypes.enum);
         } else {
-          expect(dim).toBeInstanceOf(Crossfilter.ScalarDimension);
+          expect(dim.dim).toBeInstanceOf(DimTypes.scalar);
         }
       }
     });
-    expect(dimensionMap[layoutDimensionName("XY")]).toBeInstanceOf(
-      Crossfilter.SpatialDimension
-    );
+    expect(
+      crossfilter.dimensions[layoutDimensionName("XY")].dim
+    ).toBeInstanceOf(DimTypes.spatial);
   });
 });
 
-describe("createVarDataDimension", () => {
-  /* create default universe */
-  const { world, crossfilter } = defaultBigBang();
-  world.varData = world.varData.withCol(
-    "GENE",
-    Float32Array.from(_.range(world.nObs))
-  );
-  const result = World.createVarDataDimension(world, crossfilter, "GENE");
-  expect(result).toBeInstanceOf(Crossfilter.ScalarDimension);
-});
-
 describe("worldEqUniverse", () => {
   const { universe, world } = defaultBigBang();
   const result = World.worldEqUniverse(world, universe);

client/__tests__/util/stateManager/worldUtil.test.js

@@ -1,62 +0,0 @@
-import {
-  countCategoryValues2D,
-  clearCaches
-} from "../../../src/util/stateManager/worldUtil";
-import * as Dataframe from "../../../src/util/dataframe";
-
-describe("WorldUtil cache management", () => {
-  test("empty", () => {
-    const count = countCategoryValues2D(
-      "a",
-      "b",
-      new Dataframe.Dataframe([0, 0], [])
-    );
-    expect(count).toMatchObject(new Map());
-    expect(count.size).toBe(0);
-  });
-
-  test("simple couts", () => {
-    const df = new Dataframe.Dataframe(
-      [3, 2],
-      [[0, 0, 1], [false, true, false]],
-      null,
-      new Dataframe.KeyIndex(["a", "b"])
-    );
-    const count = countCategoryValues2D("a", "b", df);
-    expect(count).toMatchObject(
-      new Map([
-        [0, new Map([[true, 1], [false, 1]])],
-        [1, new Map([[false, 1]])]
-      ])
-    );
-  });
-
-  test("memo cache clear", () => {
-    clearCaches();
-    const df1 = new Dataframe.Dataframe([0, 0], []);
-    const df2 = new Dataframe.Dataframe(
-      [3, 2],
-      [[0, 0, 1], [false, true, false]],
-      null,
-      new Dataframe.KeyIndex(["a", "b"])
-    );
-
-    const count1 = countCategoryValues2D("a", "b", df1);
-    const count2 = countCategoryValues2D("a", "b", df1);
-    const count3 = countCategoryValues2D("a", "b", df1.clone());
-    const count4 = countCategoryValues2D("a", "b", df2);
-
-    clearCaches();
-    const count10 = countCategoryValues2D("a", "b", df1);
-    const count11 = countCategoryValues2D("a", "b", df2);
-
-    expect(count1).toEqual(count2);
-    expect(count1).toEqual(count3);
-    expect(count1).toEqual(count10);
-    expect(count1).not.toBe(count3);
-    expect(count1).not.toBe(count10);
-
-    expect(count4).toEqual(count11);
-    expect(count4).not.toBe(count11);
-  });
-});

client/__tests__/util/typedCrossfilter/crossfilter.test.js

@@ -0,0 +1,434 @@
+import _ from "lodash";
+import { polygonContains } from "d3";
+
+import Crossfilter from "../../../src/util/typedCrossfilter";
+
+const someData = [
+  {
+    date: "2011-11-14T16:17:54Z",
+    quantity: 2,
+    total: 190,
+    tip: 100,
+    type: "tab",
+    productIDs: ["001"],
+    coords: [0, 0],
+    nonFinite: 0.0
+  },
+  {
+    date: "2011-11-14T16:20:19Z",
+    quantity: 2,
+    total: 190,
+    tip: 100,
+    type: "tab",
+    productIDs: ["001", "005"],
+    coords: [0.4, 0.4],
+    nonFinite: Number.NaN
+  },
+  {
+    date: "2011-11-14T16:28:54Z",
+    quantity: 1,
+    total: 300,
+    tip: 200,
+    type: "visa",
+    productIDs: ["004", "005"],
+    coords: [0.3, 0.1],
+    nonFinite: Number.POSITIVE_INFINITY
+  },
+  {
+    date: "2011-11-14T16:30:43Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["001", "002"],
+    coords: [0.392, 0.1],
+    nonFinite: Number.NEGATIVE_INFINITY
+  },
+  {
+    date: "2011-11-14T16:48:46Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["005"],
+    coords: [0.7, 0.0482],
+    nonFinite: 1.0
+  },
+  {
+    date: "2011-11-14T16:53:41Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["001", "004", "005"],
+    coords: [0.9999, 1.0],
+    nonFinite: Number.NaN
+  },
+  {
+    date: "2011-11-14T16:54:06Z",
+    quantity: 1,
+    total: 100,
+    tip: 0,
+    type: "cash",
+    productIDs: ["001", "002", "003", "004", "005"],
+    coords: [0.384, 0.6938],
+    nonFinite: 99.0
+  },
+  {
+    date: "2011-11-14T16:58:03Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["001"],
+    coords: [0.4822, 0.482],
+    nonFinite: Number.NaN
+  },
+  {
+    date: "2011-11-14T17:07:21Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["004", "005"],
+    coords: [0.2234, 0],
+    nonFinite: Number.NaN
+  },
+  {
+    date: "2011-11-14T17:22:59Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["001", "002", "004", "005"],
+    coords: [0.382, 0.38485],
+    nonFinite: -1
+  },
+  {
+    date: "2011-11-14T17:25:45Z",
+    quantity: 2,
+    total: 200,
+    tip: 0,
+    type: "cash",
+    productIDs: ["002"],
+    coords: [0.998, 0.8472],
+    nonFinite: 0.0
+  },
+  {
+    date: "2011-11-14T17:29:52Z",
+    quantity: 1,
+    total: 200,
+    tip: 100,
+    type: "visa",
+    productIDs: ["004"],
+    coords: [0.8273, 0.3384],
+    nonFinite: 0.0
+  }
+];
+
+let payments = null;
+beforeEach(() => {
+  payments = new Crossfilter(someData);
+});
+
+describe("ImmutableTypedCrossfilter", () => {
+  test("create crossfilter", () => {
+    expect(payments).toBeDefined();
+    expect(payments.size()).toEqual(someData.length);
+    expect(payments.all()).toEqual(someData);
+
+    const p = payments
+      .addDimension("quantity", "scalar", (i, d) => d[i].quantity, Int32Array)
+      .select("quantity", { mode: "all" });
+    expect(p).toBeDefined();
+    expect(p.all()).toEqual(someData);
+    expect(p.size()).toEqual(someData.length);
+    expect(p.isElementSelected(0)).toBeTruthy();
+    expect(p.countSelected()).toEqual(someData.length);
+    expect(p.allSelected()).toEqual(someData);
+  });
+
+  test("immutability", () => {
+    /*
+    the following should return a new crossfilter:
+    - addDimension()
+    - delDimension()
+    - select
+    */
+    const p2 = payments.addDimension(
+      "quantity",
+      "scalar",
+      (i, data) => data[i].quantity,
+      Int32Array
+    );
+
+    expect(payments).not.toBe(p2);
+    const p3 = p2.select("quantity", { mode: "all" });
+    expect(p3).not.toBe(p2);
+
+    const p4 = p3.delDimension("quantity");
+    expect(p4).not.toBe(p3);
+  });
+
+  test("select all and none", () => {
+    let p = payments
+      .addDimension("quantity", "scalar", (i, d) => d[i].quantity, Int32Array)
+      .addDimension("tip", "scalar", (i, d) => d[i].tip, Float32Array)
+      .addDimension("total", "scalar", (i, d) => d[i].total, Float32Array)
+      .addDimension("type", "enum", (i, d) => d[i].type);
+    expect(p).toBeDefined();
+
+    /* expect all records to be selected - default init state */
+    expect(p.allSelected()).toEqual(someData);
+    expect(p.countSelected()).toEqual(someData.length);
+    expect(p.allSelectedMask()).toEqual(
+      new Uint8Array(someData.length).fill(1)
+    );
+    expect(p.fillByIsSelected(new Uint8Array(someData.length), 99, 0)).toEqual(
+      new Uint8Array(someData.length).fill(99)
+    );
+    for (let i = 0; i < someData.length; i += 1) {
+      expect(p.isElementSelected(i)).toBeTruthy();
+    }
+
+    /* expect a selectAll on one dimension to change nothing */
+    p = p.select("tip", { mode: "all" });
+    expect(p.allSelected()).toEqual(someData);
+
+    /* ditto */
+    p = p.select("quantity", { mode: "all" });
+    expect(p.allSelected()).toEqual(someData);
+
+    /* select none on one dimension */
+    p = p.select("type", { mode: "none" });
+    expect(p.allSelected()).toEqual([]);
+    expect(p.countSelected()).toEqual(0);
+    expect(p.allSelectedMask()).toEqual(
+      new Uint8Array(someData.length).fill(0)
+    );
+    expect(p.fillByIsSelected(new Uint8Array(someData.length), 99, 0)).toEqual(
+      new Uint8Array(someData.length).fill(0)
+    );
+    for (let i = 0; i < someData.length; i += 1) {
+      expect(p.isElementSelected(i)).toBeFalsy();
+    }
+
+    p = p.select("quantity", { mode: "none" });
+    expect(p.allSelected()).toEqual([]);
+
+    // invert the first none; should have no effect because type is
+    // still not filtered.
+    p = p.select("quantity", { mode: "all" });
+    expect(p.allSelected()).toEqual([]);
+
+    /* select all of type; should select all records */
+    p = p.select("type", { mode: "all" });
+    expect(p.allSelected()).toEqual(someData);
+  });
+
+  describe("scalar dimension", () => {
+    let p;
+    beforeEach(() => {
+      p = payments
+        .addDimension("quantity", "scalar", (i, d) => d[i].quantity, Int32Array)
+        .addDimension("tip", "scalar", (i, d) => d[i].tip, Float32Array)
+        .select("tip", { mode: "all" });
+    });
+
+    /*
+    select modes:  all, none, exact, range
+    */
+    test("all", () => {
+      expect(p.select("quantity", { mode: "all" }).countSelected()).toEqual(
+        someData.length
+      );
+    });
+    test("none", () => {
+      expect(p.select("quantity", { mode: "none" }).countSelected()).toEqual(0);
+    });
+    test.each([[[]], [[2]], [[2, 1]], [[9, 82]], [[0, 1]]])("exact: %p", v =>
+      expect(
+        p.select("quantity", { mode: "exact", values: v }).countSelected()
+      ).toEqual(_.filter(someData, d => v.includes(d.quantity)).length)
+    );
+    test.each([[0, 1], [1, 2], [0, 99], [99, 100000]])("range %p", (lo, hi) =>
+      expect(
+        p.select("quantity", { mode: "range", lo, hi }).countSelected()
+      ).toEqual(
+        _.filter(someData, d => d.quantity >= lo && d.quantity < hi).length
+      )
+    );
+    test("bad mode", () => {
+      expect(() => p.select("type", { mode: "bad mode" })).toThrow(Error);
+    });
+  });
+
+  describe("enum dimension", () => {
+    let p;
+    beforeEach(() => {
+      p = payments.addDimension("type", "enum", (i, d) => d[i].type);
+    });
+
+    test("all", () => {
+      expect(p.select("type", { mode: "all" }).countSelected()).toEqual(
+        someData.length
+      );
+    });
+    test("none", () => {
+      expect(p.select("type", { mode: "none" }).countSelected()).toEqual(0);
+    });
+    test.each([
+      [[]],
+      [["tab"]],
+      [["visa"]],
+      [["visa", "tab"]],
+      [["cash", "tab", "visa"]]
+    ])("exact: %p", v =>
+      expect(
+        p.select("type", { mode: "exact", values: v }).countSelected()
+      ).toEqual(_.filter(someData, d => v.includes(d.type)).length)
+    );
+    test("range", () => {
+      expect(() => p.select("type", { mode: "range", lo: 0, hi: 9 })).toThrow(
+        Error
+      );
+    });
+    test("bad mode", () => {
+      expect(() => p.select("type", { mode: "bad mode" })).toThrow(Error);
+    });
+  });
+
+  describe("spatial dimension", () => {
+    let p;
+    beforeEach(() => {
+      const X = someData.map(r => r.coords[0]);
+      const Y = someData.map(r => r.coords[1]);
+      p = payments.addDimension("coords", "spatial", X, Y);
+    });
+
+    test("all", () => {
+      expect(p.select("coords", { mode: "all" }).countSelected()).toEqual(
+        someData.length
+      );
+    });
+    test("none", () => {
+      expect(p.select("coords", { mode: "none" }).countSelected()).toEqual(0);
+    });
+    test.each([[0, 0, 1, 1], [0, 0, 0.5, 0.5], [0.5, 0.5, 1, 1]])(
+      "within-rect %d %d %d %d",
+      (minX, minY, maxX, maxY) => {
+        expect(
+          p
+            .select("coords", { mode: "within-rect", minX, minY, maxX, maxY })
+            .allSelected()
+        ).toEqual(
+          _.filter(someData, d => {
+            const [x, y] = d.coords;
+            return minX <= x && x < maxX && minY <= y && y < maxY;
+          })
+        );
+      }
+    );
+
+    test.each([
+      [[[0, 0], [0, 1], [1, 1], [1, 0]]],
+      [[[0, 0], [0, 0.5], [0.5, 0.5], [0.5, 0]]]
+    ])("within-polygon %p", polygon => {
+      expect(
+        p.select("coords", { mode: "within-polygon", polygon }).allSelected()
+      ).toEqual(_.filter(someData, d => polygonContains(polygon, d.coords)));
+    });
+  });
+
+  describe("non-finite scalars", () => {
+    let p;
+    beforeEach(() => {
+      p = payments
+        .addDimension("quantity", "scalar", (i, d) => d[i].quantity, Int32Array)
+        .addDimension(
+          "nonFinite",
+          "scalar",
+          (i, d) => d[i].nonFinite,
+          Float32Array
+        )
+        .select("quantity", { mode: "all" });
+    });
+
+    test("all or none", () => {
+      expect(p.select("nonFinite", { mode: "all" }).countSelected()).toEqual(
+        someData.length
+      );
+      expect(p.select("nonFinite", { mode: "none" }).countSelected()).toEqual(
+        0
+      );
+    });
+
+    test("exact", () => {
+      expect(
+        p.select("nonFinite", { mode: "exact", values: [0] }).countSelected()
+      ).toEqual(3);
+      expect(
+        p.select("nonFinite", { mode: "exact", values: [1] }).countSelected()
+      ).toEqual(1);
+      expect(
+        p
+          .select("nonFinite", {
+            mode: "exact",
+            values: [Number.POSITIVE_INFINITY]
+          })
+          .countSelected()
+      ).toEqual(1);
+      expect(
+        p
+          .select("nonFinite", {
+            mode: "exact",
+            values: [Number.NEGATIVE_INFINITY]
+          })
+          .countSelected()
+      ).toEqual(1);
+      expect(
+        p
+          .select("nonFinite", { mode: "exact", values: [Number.NaN] })
+          .countSelected()
+      ).toEqual(4);
+      expect(
+        p
+          .select("nonFinite", {
+            mode: "exact",
+            values: [Number.POSITIVE_INFINITY, 0, 1, 99]
+          })
+          .countSelected()
+      ).toEqual(6);
+    });
+
+    test("range", () => {
+      expect(
+        p
+          .select("nonFinite", {
+            mode: "range",
+            lo: 0,
+            hi: Number.POSITIVE_INFINITY
+          })
+          .countSelected()
+      ).toEqual(5);
+      expect(
+        p
+          .select("nonFinite", {
+            mode: "range",
+            lo: 0,
+            hi: Number.NaN
+          })
+          .countSelected()
+      ).toEqual(6);
+      expect(
+        p
+          .select("nonFinite", {
+            mode: "range",
+            lo: Number.NEGATIVE_INFINITY,
+            hi: Number.POSITIVE_INFINITY
+          })
+          .countSelected()
+      ).toEqual(7);
+    });
+  });
+});

client/__tests__/util/typedCrossfilter/sort.test.js

@@ -1,4 +1,22 @@
-import { sort, sortIndex } from "../../../src/util/typedCrossfilter/sort";
+import {
+  sortArray,
+  sortIndex,
+  lowerBound,
+  upperBound,
+  lowerBoundIndirect,
+  upperBoundIndirect
+} from "../../../src/util/typedCrossfilter/sort";
+
+/*
+Sort tests should keep in mind that there are separate code
+paths for:
+  - small vs. large arrays (insertionsort only)
+  - float-only typed arrays vs. other array types (non-finite handling)
+  - indexed vs. direct sort
+*/
+
+const pInf = Number.POSITIVE_INFINITY;
+const nInf = Number.NEGATIVE_INFINITY;
 
 function fillRange(arr, start = 0) {
   const larr = arr;
@@ -15,42 +33,225 @@ function fillRand(arr) {
   return arr;
 }
 
-describe("sort", () => {
-  [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map(Type =>
-    test(Type.name, () => {
-      expect(sort(Type.from([6, 5, 4, 3, 2, 1, 0]))).toMatchObject(
-        Type.from([0, 1, 2, 3, 4, 5, 6])
-      );
-      expect(sort(Type.from([6, 5, 4, 3, 2, 1]))).toMatchObject(
-        Type.from([1, 2, 3, 4, 5, 6])
-      );
+describe("sortArray", () => {
+  describe("JS vals", () => {
+    [
+      [true, false],
+      ["a", "b", "0", "1"],
+      [0, "a", true, null, undefined, 3.1415],
+      fillRand(new Array(1000)),
+      ["a", NaN, null, pInf]
+    ].map((val, idx) =>
+      test(`JS vals ${idx}`, () => {
+        expect(sortArray(val)).toMatchObject(val.sort());
+      })
+    );
+  });
 
-      const source = fillRand(new Type(1000));
-      expect(sort(Type.from(source))).toMatchObject(Type.from(source).sort());
-    })
-  );
+  describe("finite numbers", () => {
+    [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map(Type =>
+      test(Type.name, () => {
+        expect(sortArray(Type.from([6, 5, 4, 3, 2, 1, 0]))).toMatchObject(
+          Type.from([0, 1, 2, 3, 4, 5, 6])
+        );
+
+        expect(sortArray(Type.from([6, 5, 4, 3, 2, 1]))).toMatchObject(
+          Type.from([1, 2, 3, 4, 5, 6])
+        );
+
+        const source = fillRand(new Type(1000));
+        expect(sortArray(Type.from(source))).toMatchObject(
+          Type.from(source).sort()
+        );
+      })
+    );
+  });
+
+  describe("non-finite numbers", () => {
+    test("inifinity", () => {
+      expect(sortArray(new Float32Array([pInf, nInf, 0, 1, 2]))).toMatchObject(
+        new Float32Array([nInf, 0, 1, 2, pInf])
+      );
+      expect(
+        sortArray(new Float32Array([pInf, nInf, pInf, nInf]))
+      ).toMatchObject(new Float32Array([nInf, nInf, pInf, pInf]));
+      expect(
+        sortArray(new Float32Array([pInf, nInf, pInf, nInf, pInf]))
+      ).toMatchObject(new Float32Array([nInf, nInf, pInf, pInf, pInf]));
+      expect(
+        sortArray(
+          new Float32Array(100).fill(Infinity, 0, 50).fill(-Infinity, 50, 100)
+        )
+      ).toMatchObject(
+        new Float32Array(100).fill(-Infinity, 0, 50).fill(Infinity, 50, 100)
+      );
+    });
+    test("NaN", () => {
+      expect(sortArray(new Float64Array([NaN, 2, 1, 0]))).toMatchObject(
+        new Float64Array([0, 1, 2, NaN])
+      );
+      expect(sortArray(new Float32Array([NaN, 2, 1, 0]))).toMatchObject(
+        new Float32Array([0, 1, 2, NaN])
+      );
+      expect(sortArray(new Float32Array([NaN, 2, NaN, 1, 0]))).toMatchObject(
+        new Float32Array([0, 1, 2, NaN, NaN])
+      );
+      expect(sortArray(new Float32Array([NaN, 2, 1, NaN, 0]))).toMatchObject(
+        new Float32Array([0, 1, 2, NaN, NaN])
+      );
+      expect(
+        sortArray(fillRange(new Float32Array(100)).fill(NaN, 0, 10))
+      ).toMatchObject(fillRange(new Float32Array(100), 10).fill(NaN, 90, 100));
+    });
+
+    test("mixed numbers", () => {
+      expect(
+        sortArray(new Float32Array([NaN, pInf, nInf, NaN, NaN]))
+      ).toMatchObject(new Float32Array([nInf, pInf, NaN, NaN, NaN]));
+      expect(
+        sortArray(new Float32Array([NaN, pInf, nInf, NaN, 1, NaN, 2]))
+      ).toMatchObject(new Float32Array([nInf, 1, 2, pInf, NaN, NaN, NaN]));
+      expect(
+        sortArray(new Float32Array([NaN, pInf, nInf, 0, 1, NaN, 2]))
+      ).toMatchObject(new Float32Array([nInf, 0, 1, 2, pInf, NaN, NaN]));
+      expect(
+        sortArray(
+          fillRange(new Float32Array(100))
+            .fill(NaN, 0, 10)
+            .fill(Infinity, 10, 20)
+        )
+      ).toMatchObject(
+        fillRange(new Float32Array(100), 20)
+          .fill(Infinity, 80, 90)
+          .fill(NaN, 90, 100)
+      );
+    });
+  });
 });
 
 describe("sortIndex", () => {
-  [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map(Type =>
-    test(Type.name, () => {
-      const source1 = Type.from([6, 5, 4, 3, 2, 1, 0]);
+  describe("finite numbers", () => {
+    [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map(Type =>
+      test(Type.name, () => {
+        const source1 = Type.from([6, 5, 4, 3, 2, 1, 0]);
+        const index1 = fillRange(new Uint32Array(source1.length));
+        expect(sortIndex(index1, source1)).toMatchObject(
+          index1.sort((a, b) => source1[a] - source1[b])
+        );
+
+        const source2 = Type.from([6, 5, 4, 3, 2, 1]);
+        const index2 = fillRange(new Uint32Array(source2.length));
+        expect(sortIndex(index2, source2)).toMatchObject(
+          index2.sort((a, b) => source1[a] - source1[b])
+        );
+
+        const source3 = fillRand(new Type(1000));
+        const index3 = fillRange(new Uint32Array(source3.length));
+        expect(sortIndex(index3, source3)).toMatchObject(
+          index3.sort((a, b) => source1[a] - source1[b])
+        );
+      })
+    );
+  });
+
+  describe("non-finite numbers", () => {
+    test("mixed numbers", () => {
+      const source1 = new Float32Array([NaN, pInf, nInf, NaN, 1, NaN, 2]);
       const index1 = fillRange(new Uint32Array(source1.length));
       expect(sortIndex(index1, source1)).toMatchObject(
-        index1.sort((a, b) => source1[a] - source1[b])
+        new Uint32Array([2, 4, 6, 1, 0, 3, 5])
       );
 
-      const source2 = Type.from([6, 5, 4, 3, 2, 1]);
+      const source2 = new Float32Array([NaN, pInf, nInf, 0, 1, NaN, 2]);
       const index2 = fillRange(new Uint32Array(source2.length));
       expect(sortIndex(index2, source2)).toMatchObject(
-        index2.sort((a, b) => source1[a] - source1[b])
+        new Uint32Array([2, 3, 4, 6, 1, 0, 5])
       );
+    });
+  });
+});
 
-      const source3 = fillRand(new Type(1000));
-      const index3 = fillRange(new Uint32Array(source3.length));
-      expect(sortIndex(index3, source3)).toMatchObject(
-        index3.sort((a, b) => source1[a] - source1[b])
-      );
-    })
-  );
+describe("lowerBound", () => {
+  test("non-float path", () => {
+    expect(lowerBound([], 0, 0, 0)).toEqual(0);
+
+    expect(lowerBound([0, 1, 2, 3], -1, 0, 4)).toEqual(0);
+    expect(lowerBound([0, 1, 2, 3], 0, 0, 4)).toEqual(0);
+    expect(lowerBound([0, 1, 2, 3], 1, 0, 4)).toEqual(1);
+    expect(lowerBound([0, 1, 2, 3], 3, 0, 4)).toEqual(3);
+    expect(lowerBound([0, 1, 2, 3], 4, 0, 4)).toEqual(4);
+    expect(lowerBound([0, 1, 2, 3], 4, 0, 3)).toEqual(3);
+
+    expect(lowerBound([0, 1, 2, 3, 4], -1, 0, 5)).toEqual(0);
+    expect(lowerBound([0, 1, 2, 3, 4], 0, 0, 5)).toEqual(0);
+    expect(lowerBound([0, 1, 2, 3, 4], 2, 0, 5)).toEqual(2);
+    expect(lowerBound([0, 1, 2, 3, 4], 4, 0, 5)).toEqual(4);
+    expect(lowerBound([0, 1, 2, 3, 4], 5, 0, 5)).toEqual(5);
+
+    expect(lowerBound([0, 2, 4, 6, 8], 5, 0, 5)).toEqual(3);
+    expect(lowerBound([0, 2, 2, 2, 8], 5, 0, 5)).toEqual(4);
+
+    expect(lowerBound([0, 1, 2, 3, 4, 5, 6, 7, 8], 3, 2, 4)).toEqual(3);
+    expect(lowerBound([0, 1, 2, 3, 4, 5, 6, 7, 8], 99, 2, 4)).toEqual(4);
+  });
+
+  test("float path, finites", () => {
+    expect(lowerBound(new Float32Array([0, 1, 2, 3]), 1, 0, 4)).toEqual(1);
+
+    expect(lowerBound(new Float32Array([]), 0, 0, 0)).toEqual(0);
+
+    expect(lowerBound(new Float32Array([0, 1, 2, 3]), -1, 0, 4)).toEqual(0);
+    expect(lowerBound(new Float32Array([0, 1, 2, 3]), 0, 0, 4)).toEqual(0);
+    expect(lowerBound(new Float32Array([0, 1, 2, 3]), 1, 0, 4)).toEqual(1);
+    expect(lowerBound(new Float32Array([0, 1, 2, 3]), 3, 0, 4)).toEqual(3);
+    expect(lowerBound(new Float32Array([0, 1, 2, 3]), 4, 0, 4)).toEqual(4);
+    expect(lowerBound(new Float32Array([0, 1, 2, 3]), 4, 0, 3)).toEqual(3);
+
+    expect(lowerBound(new Float32Array([0, 1, 2, 3, 4]), -1, 0, 5)).toEqual(0);
+    expect(lowerBound(new Float32Array([0, 1, 2, 3, 4]), 0, 0, 5)).toEqual(0);
+    expect(lowerBound(new Float32Array([0, 1, 2, 3, 4]), 2, 0, 5)).toEqual(2);
+    expect(lowerBound(new Float32Array([0, 1, 2, 3, 4]), 4, 0, 5)).toEqual(4);
+    expect(lowerBound(new Float32Array([0, 1, 2, 3, 4]), 5, 0, 5)).toEqual(5);
+
+    expect(lowerBound(new Float32Array([0, 2, 4, 6, 8]), 5, 0, 5)).toEqual(3);
+    expect(lowerBound(new Float32Array([0, 2, 2, 2, 8]), 5, 0, 5)).toEqual(4);
+
+    expect(
+      lowerBound(new Float32Array([0, 1, 2, 3, 4, 5, 6, 7, 8]), 3, 2, 4)
+    ).toEqual(3);
+    expect(
+      lowerBound(new Float32Array([0, 1, 2, 3, 4, 5, 6, 7, 8]), 99, 2, 4)
+    ).toEqual(4);
+  });
+
+  test("float path, non-finite", () => {
+    expect(
+      lowerBound(
+        new Float32Array([-Infinity, 0, 1, Infinity, NaN]),
+        -Infinity,
+        0,
+        5
+      )
+    ).toEqual(0);
+    expect(
+      lowerBound(new Float32Array([-Infinity, 0, 1, Infinity, NaN]), 0, 0, 5)
+    ).toEqual(1);
+    expect(
+      lowerBound(new Float32Array([-Infinity, 0, 1, Infinity, NaN]), 1, 0, 5)
+    ).toEqual(2);
+    expect(
+      lowerBound(new Float32Array([-Infinity, 0, 1, Infinity, NaN]), 2, 0, 5)
+    ).toEqual(3);
+    expect(
+      lowerBound(
+        new Float32Array([-Infinity, 0, 1, Infinity, NaN]),
+        Infinity,
+        0,
+        5
+      )
+    ).toEqual(3);
+    expect(
+      lowerBound(new Float32Array([-Infinity, 0, 1, Infinity, NaN]), NaN, 0, 5)
+    ).toEqual(4);
+  });
 });

client/__tests__/util/typedCrossfilter/typedCrossfilter.test.js

@@ -1,590 +0,0 @@
-// jshint esversion: 6
-import _ from "lodash";
-import crossfilter from "../../../src/util/typedCrossfilter";
-
-const someData = [
-  {
-    date: "2011-11-14T16:17:54Z",
-    quantity: 2,
-    total: 190,
-    tip: 100,
-    type: "tab",
-    productIDs: ["001"]
-  },
-  {
-    date: "2011-11-14T16:20:19Z",
-    quantity: 2,
-    total: 190,
-    tip: 100,
-    type: "tab",
-    productIDs: ["001", "005"]
-  },
-  {
-    date: "2011-11-14T16:28:54Z",
-    quantity: 1,
-    total: 300,
-    tip: 200,
-    type: "visa",
-    productIDs: ["004", "005"]
-  },
-  {
-    date: "2011-11-14T16:30:43Z",
-    quantity: 2,
-    total: 90,
-    tip: 0,
-    type: "tab",
-    productIDs: ["001", "002"]
-  },
-  {
-    date: "2011-11-14T16:48:46Z",
-    quantity: 2,
-    total: 90,
-    tip: 0,
-    type: "tab",
-    productIDs: ["005"]
-  },
-  {
-    date: "2011-11-14T16:53:41Z",
-    quantity: 2,
-    total: 90,
-    tip: 0,
-    type: "tab",
-    productIDs: ["001", "004", "005"]
-  },
-  {
-    date: "2011-11-14T16:54:06Z",
-    quantity: 1,
-    total: 100,
-    tip: 0,
-    type: "cash",
-    productIDs: ["001", "002", "003", "004", "005"]
-  },
-  {
-    date: "2011-11-14T16:58:03Z",
-    quantity: 2,
-    total: 90,
-    tip: 0,
-    type: "tab",
-    productIDs: ["001"]
-  },
-  {
-    date: "2011-11-14T17:07:21Z",
-    quantity: 2,
-    total: 90,
-    tip: 0,
-    type: "tab",
-    productIDs: ["004", "005"]
-  },
-  {
-    date: "2011-11-14T17:22:59Z",
-    quantity: 2,
-    total: 90,
-    tip: 0,
-    type: "tab",
-    productIDs: ["001", "002", "004", "005"]
-  },
-  {
-    date: "2011-11-14T17:25:45Z",
-    quantity: 2,
-    total: 200,
-    tip: 0,
-    type: "cash",
-    productIDs: ["002"]
-  },
-  {
-    date: "2011-11-14T17:29:52Z",
-    quantity: 1,
-    total: 200,
-    tip: 100,
-    type: "visa",
-    productIDs: ["004"]
-  }
-];
-
-function groupReduce(data, valueMap, valueReduce, valueInit) {
-  return _.reduce(
-    data,
-    (acc, value) => {
-      const k = valueMap(value);
-      let r = _.find(acc, o => o.key === k);
-      if (!r) {
-        r = { key: k, value: valueInit() };
-        acc.push(r);
-      }
-      r.value = valueReduce(r.value, value);
-      return acc;
-    },
-    []
-  ).sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
-}
-
-function groupCount(data, map) {
-  return groupReduce(data, map, p => p + 1, () => 0);
-}
-
-function groupSum(data, map) {
-  return groupReduce(
-    data,
-    map,
-    (p, v) => {
-      p += map(v);
-      return p;
-    },
-    () => 0
-  );
-}
-
-let payments = null;
-beforeEach(() => {
-  payments = crossfilter(someData);
-});
-
-describe("typedCrossfilter", () => {
-  test("alloc and free", () => {
-    expect(payments).toBeDefined();
-    expect(payments.size()).toEqual(someData.length);
-    expect(payments.all()).toEqual(someData);
-
-    const quantity = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].quantity,
-      Int32Array
-    );
-    expect(quantity).toBeDefined();
-    expect(quantity.id()).toBeDefined();
-
-    quantity.dispose();
-    expect(payments.size()).toEqual(someData.length);
-    expect(payments.all()).toEqual(someData);
-  });
-
-  test("filterAll and filterNone", () => {
-    expect(payments).toBeDefined();
-    const quantity = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].quantity,
-      Int32Array
-    );
-    const tip = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].tip,
-      Float32Array
-    );
-    const total = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].total,
-      Float32Array
-    );
-    const type = payments.dimension(
-      crossfilter.EnumDimension,
-      (i, data) => data[i].type
-    );
-
-    expect(quantity).toBeDefined();
-    expect(tip).toBeDefined();
-    expect(total).toBeDefined();
-    expect(type).toBeDefined();
-
-    // initially, all should be filtered
-    expect(payments.allFiltered()).toHaveLength(payments.size());
-    expect(payments.allFiltered()).toEqual(payments.all());
-    expect(payments.countFiltered()).toEqual(someData.length);
-
-    // filterAll
-    tip.filterAll(); // should change nothing
-    expect(payments.allFiltered()).toEqual(payments.all());
-    expect(payments.countFiltered()).toEqual(someData.length);
-
-    // ditto
-    total.filterAll();
-    expect(payments.allFiltered()).toEqual(payments.all());
-    expect(payments.countFiltered()).toEqual(someData.length);
-
-    // filterNone
-    type.filterNone();
-    expect(payments.allFiltered()).toEqual([]);
-    expect(payments.countFiltered()).toEqual(0);
-
-    quantity.filterNone();
-    expect(payments.allFiltered()).toEqual([]);
-    expect(payments.countFiltered()).toEqual(0);
-
-    // invert the first none; should have no effect because type is
-    // still not filtered
-    quantity.filterAll();
-    expect(payments.allFiltered()).toEqual([]);
-    expect(payments.countFiltered()).toEqual(0);
-
-    // filter all of type; should select all
-    type.filterAll();
-    expect(payments.allFiltered()).toEqual(payments.all());
-    expect(payments.countFiltered()).toEqual(payments.size());
-  });
-
-  test("filterExact", () => {
-    expect(payments).toBeDefined();
-    const quantity = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].quantity,
-      Int32Array
-    );
-    const tip = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].tip,
-      Float32Array
-    );
-    const type = payments.dimension(
-      crossfilter.EnumDimension,
-      (i, data) => data[i].type
-    );
-
-    quantity.filterExact(1);
-    expect(payments.countFiltered()).toEqual(
-      _.countBy(someData, "quantity")[1]
-    );
-    expect(payments.allFiltered()).toEqual(_.filter(someData, { quantity: 1 }));
-
-    tip.filterExact(0);
-    expect(payments.allFiltered()).toEqual(
-      _.filter(someData, { tip: 0, quantity: 1 })
-    );
-
-    type.filterExact("cash");
-    expect(payments.allFiltered()).toEqual(
-      _.filter(someData, { tip: 0, quantity: 1, type: "cash" })
-    );
-  });
-
-  test("filterRange", () => {
-    expect(payments).toBeDefined();
-    const quantity = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].quantity,
-      Int32Array
-    );
-    const tip = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].tip,
-      Float32Array
-    );
-    const total = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].total,
-      Float32Array
-    );
-    const type = payments.dimension(
-      crossfilter.EnumDimension,
-      (i, data) => data[i].type
-    );
-
-    tip.filterRange([0, 91]);
-    expect(payments.allFiltered()).toEqual(
-      _(someData)
-        .filter(r => r.tip >= 0 && r.tip < 91)
-        .value()
-    );
-
-    tip.filterRange([0, 90]);
-    expect(payments.allFiltered()).toEqual(
-      _(someData)
-        .filter(r => r.tip >= 0 && r.tip < 90)
-        .value()
-    );
-
-    tip.filterRange([1, 90]);
-    expect(payments.allFiltered()).toEqual(
-      _(someData)
-        .filter(r => r.tip >= 1 && r.tip < 91)
-        .value()
-    );
-  });
-
-  test("filterEnum", () => {
-    expect(payments).toBeDefined();
-    const quantity = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].quantity,
-      Int32Array
-    );
-    const tip = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].tip,
-      Float32Array
-    );
-    const total = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].total,
-      Float32Array
-    );
-    const type = payments.dimension(
-      crossfilter.EnumDimension,
-      (i, data) => data[i].type
-    );
-
-    type.filterEnum(["tab", "cash"]);
-    expect(payments.allFiltered()).toEqual(
-      _(someData)
-        .filter(r => r.type === "cash" || r.type === "tab")
-        .value()
-    );
-
-    tip.filterEnum([0, 100]);
-    expect(payments.allFiltered()).toEqual(
-      _(someData)
-        .filter(r => r.type === "cash" || r.type === "tab")
-        .filter(r => r.tip === 0 || r.tip === 100)
-        .value()
-    );
-  });
-
-  test("more than 32 dimensions", () => {
-    expect(payments).toBeDefined();
-    const quantity = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].quantity,
-      Int32Array
-    );
-    const tip = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].tip,
-      Float32Array
-    );
-    const total = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].total,
-      Float32Array
-    );
-    const type = payments.dimension(
-      crossfilter.EnumDimension,
-      (i, data) => data[i].type
-    );
-
-    // Create a bunch of fake dimensions to ensure we can handle > 32
-    let dimMap = {};
-    for (let i = 0; i < 65; i++) {
-      dimMap[i] = payments.dimension(
-        crossfilter.ScalarDimension,
-        () => Math.random(),
-        Float32Array
-      );
-      expect(dimMap[i]).toBeDefined();
-      expect(dimMap[i].id()).toBeDefined();
-    }
-
-    // everything should start as selected/filtered
-    expect(payments.countFiltered()).toEqual(someData.length);
-
-    dimMap[0].filterAll();
-    dimMap[64].filterAll();
-    expect(payments.countFiltered()).toEqual(someData.length);
-
-    dimMap[33].filterNone();
-    expect(payments.allFiltered()).toEqual([]);
-
-    dimMap[33].filterAll();
-    expect(payments.allFiltered()).toEqual(someData);
-  });
-
-  test("group, default mapping, default reducer, no filter", () => {
-    expect(payments).toBeDefined();
-
-    const quantity = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].quantity,
-      Int32Array
-    );
-    const tip = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].tip,
-      Int32Array
-    );
-    const type = payments.dimension(
-      crossfilter.EnumDimension,
-      (i, data) => data[i].type
-    );
-    const total = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].total,
-      Int32Array
-    );
-
-    _.each(
-      {
-        tip: tip.group(r => r),
-        type: type.group(),
-        total: total.group(),
-        quantity: quantity.group()
-      },
-      (grp, k) => {
-        const whatWeExpect = groupCount(someData, v => v[k]);
-        expect(grp.all()).toEqual(whatWeExpect);
-        expect(grp.size()).toEqual(whatWeExpect.length);
-        expect(grp.dispose()).toEqual(grp);
-      }
-    );
-  });
-
-  test("group, custom map, default reducer, no filters", () => {
-    expect(payments).toBeDefined();
-
-    // custom mapping in groups only works for scalar types.  Enums do not
-    // currently implement it.
-
-    const tip = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].tip,
-      Int32Array
-    );
-    const totalX10 = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].total * 10,
-      Int32Array
-    );
-    const type = payments.dimension(
-      crossfilter.EnumDimension,
-      (i, data) => data[i].type
-    );
-
-    const paymentsByTip_A = tip.group();
-    const paymentsByTip_B = tip.group(r => 10 * r);
-    const paymentsByType = type.group(); // identity only
-    const paymentsByTotalX10_A = totalX10.group();
-    const paymentsByTotalX10_B = totalX10.group(r => r / 10);
-
-    expect(paymentsByTip_A.all()).toEqual(groupCount(someData, v => v.tip));
-    expect(paymentsByTip_B.all()).toEqual(
-      groupCount(someData, v => 10 * v.tip)
-    );
-    expect(paymentsByType.all()).toEqual(groupCount(someData, v => v.type));
-    expect(paymentsByTotalX10_A.all()).toEqual(
-      groupCount(someData, v => 10 * v.total)
-    );
-    expect(paymentsByTotalX10_B.all()).toEqual(
-      groupCount(someData, v => (10 * v.total) / 10)
-    );
-
-    for (let i of [
-      paymentsByTip_A,
-      paymentsByTip_B,
-      paymentsByType,
-      paymentsByTotalX10_A,
-      paymentsByTotalX10_B,
-      tip,
-      totalX10,
-      type
-    ]) {
-      expect(i.dispose()).toEqual(i);
-    }
-  });
-
-  test("group, default map, custom reducer, no filters", () => {
-    expect(payments).toBeDefined();
-
-    const total = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].total,
-      Float32Array
-    );
-    const type = payments.dimension(
-      crossfilter.EnumDimension,
-      (i, data) => data[i].type
-    );
-
-    const paymentsByTotal = total.group();
-    const paymentsByType = type.group();
-
-    // reduceCount
-    expect(paymentsByTotal.reduceCount()).toEqual(paymentsByTotal);
-    expect(paymentsByTotal.all()).toEqual(groupCount(someData, v => v.total));
-
-    // reduceSum
-    expect(paymentsByTotal.reduceSum(v => v.total)).toEqual(paymentsByTotal);
-    expect(paymentsByTotal.all()).toEqual(groupSum(someData, v => v.total));
-
-    // use custom reducers (my reducers) - count by three, init 1
-    expect(
-      paymentsByTotal.reduce((p, v) => (p += 3), (p, v) => (p -= 3), () => 1)
-    ).toEqual(paymentsByTotal);
-    expect(paymentsByTotal.all()).toEqual(
-      groupReduce(someData, v => v.total, (p, v) => p + 3, () => 1)
-    );
-
-    for (let i of [paymentsByTotal, paymentsByType, type]) {
-      expect(i.dispose()).toEqual(i);
-    }
-  });
-
-  test("group, default map, default reducer, filters", () => {
-    // From the docs:
-    // Note: a grouping intersects the crossfilter's current filters, except for the
-    // associated dimension's filter. Thus, group methods consider only records that
-    // satisfy every filter except this dimension's filter. So, if the crossfilter of
-    // payments is filtered by type and total, then group by total only observes the
-    // filter by type.
-
-    expect(payments).toBeDefined();
-
-    const tip = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].tip,
-      Int32Array
-    );
-    const total = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].total,
-      Int32Array
-    );
-    const type = payments.dimension(
-      crossfilter.EnumDimension,
-      (i, data) => data[i].type
-    );
-
-    const paymentsByTip = tip.group();
-    const paymentsByTotal = total.group();
-    const paymentsByType = type.group();
-
-    // 1. confirm that changing the filter on a dimension does NOT change that
-    // dimensions groups.
-    {
-      tip.filterAll(), total.filterAll(), type.filterAll();
-      let before = _.cloneDeep(paymentsByTip.all());
-      tip.filterExact(0);
-      expect(paymentsByTip.all()).toEqual(before);
-    }
-
-    // 2. confirm that changing a filter on a different dimension DOES change
-    // all other groups.
-    {
-      tip.filterAll(), total.filterAll(), type.filterAll();
-      const before = _.cloneDeep([paymentsByTotal.all(), paymentsByType.all()]);
-      tip.filterExact(0);
-      const after = [paymentsByTotal.all(), paymentsByType.all()];
-      expect(after).not.toEqual(before);
-      expect(after).toEqual([
-        groupReduce(
-          someData,
-          v => v.total,
-          (p, v) => (v.tip !== 0 ? p : p + 1),
-          () => 0
-        ),
-        groupReduce(
-          someData,
-          v => v.type,
-          (p, v) => (v.tip !== 0 ? p : p + 1),
-          () => 0
-        )
-      ]);
-    }
-
-    for (let i of [
-      paymentsByTip,
-      paymentsByTotal,
-      paymentsByType,
-      tip,
-      total,
-      type
-    ]) {
-      expect(i.dispose()).toEqual(i);
-    }
-  });
-});

client/__tests__/util/typedCrossfilter/util.test.js

@@ -1,11 +1,8 @@
 import {
-  fillRange,
   sliceByIndex,
-  makeSortIndex,
-  lowerBound,
-  lowerBoundIndirect,
-  upperBoundIndirect
+  makeSortIndex
 } from "../../../src/util/typedCrossfilter/util";
+import { rangeFill as fillRange } from "../../../src/util/range";
 
 describe("fillRange", () => {
   test("Array", () => {

client/configuration/babel/babel.dev.js

@@ -7,8 +7,10 @@ module.exports = {
   ],
   plugins: [
     "@babel/plugin-proposal-function-bind",
-    "@babel/plugin-proposal-class-properties",
     ["@babel/plugin-proposal-decorators", { legacy: true }],
-    "@babel/plugin-proposal-export-namespace-from"
+    ["@babel/plugin-proposal-class-properties", { loose: true }],
+    "@babel/plugin-proposal-export-namespace-from",
+    "@babel/plugin-proposal-optional-chaining",
+    "@babel/plugin-proposal-nullish-coalescing-operator"
   ]
 };

client/configuration/babel/babel.prod.js

@@ -6,10 +6,12 @@ module.exports = {
   ],
   plugins: [
     "@babel/plugin-proposal-function-bind",
-    "@babel/plugin-proposal-class-properties",
     ["@babel/plugin-proposal-decorators", { legacy: true }],
+    ["@babel/plugin-proposal-class-properties", { loose: true }],
     "@babel/plugin-proposal-export-namespace-from",
     "@babel/plugin-transform-react-constant-elements",
-    "@babel/plugin-transform-runtime"
+    "@babel/plugin-transform-runtime",
+    "@babel/plugin-proposal-optional-chaining",
+    "@babel/plugin-proposal-nullish-coalescing-operator"
   ]
 };

client/configuration/webpack/webpack.config.dev.js

@@ -37,8 +37,9 @@ module.exports = {
           {
             loader: "css-loader",
             options: {
-              modules: true,
-              localIdentName: "[name]__[local]___[hash:base64:5]"
+              modules: {
+                localIdentName: "[name]__[local]___[hash:base64:5]"
+              }
             }
           }
         ]
@@ -77,6 +78,9 @@ module.exports = {
       template: path.resolve("index.html"),
       favicon: path.resolve("favicon.png")
     }),
-    new webpack.NoEmitOnErrorsPlugin()
+    new webpack.NoEmitOnErrorsPlugin(),
+    new webpack.DefinePlugin({
+      __REACT_DEVTOOLS_GLOBAL_HOOK__: "({ isDisabled: true })"
+    })
   ]
 };

client/configuration/webpack/webpack.config.prod.js

@@ -40,9 +40,10 @@ module.exports = {
           {
             loader: "css-loader",
             options: {
-              modules: true,
-              importLoaders: 1,
-              localIdentName: "[name]__[local]___[hash:base64:5]"
+              modules: {
+                localIdentName: "[name]__[local]___[hash:base64:5]"
+              },
+              importLoaders: 1
             }
           }
         ]

client/index.html

@@ -3,7 +3,7 @@
   <head>
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1">
-    <title>cellxgene</title>
+    <title>cell&times;gene</title>
     <link href="https://fonts.googleapis.com/css?family=Roboto+Condensed:400,400i,700" rel="stylesheet">
     <style>
       html, body, p, h1, h2, h3, h4, h5, h6, span, button, input, label, text, div {

client/index_template.html

@@ -1,35 +1,42 @@
 <!DOCTYPE html>
 <html lang="en">
-  <head>
+<head>
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1">
-    <title>cellxgene</title>
+    <title>cell&times;gene</title>
     <link href="https://fonts.googleapis.com/css?family=Roboto+Condensed:400,400i,700" rel="stylesheet">
     <style>
-      html, body, p, h1, h2, h3, h4, h5, h6, span, button, input, label, text, div {
-        font-family: 'Roboto Condensed','Helvetica Neue','Helvetica','Arial',sans-serif;
-        font-size: 14px;
-      }
-      body {
-        margin: 0;
-        padding: 0;
-      }
+        html, body, p, h1, h2, h3, h4, h5, h6, span, button, input, label, text, div {
+            font-family: 'Roboto Condensed', 'Helvetica Neue', 'Helvetica', 'Arial', sans-serif;
+            font-size: 14px;
+        }
 
-      * {
-        box-sizing: border-box;
-      }
+        body {
+            margin: 0;
+            padding: 0;
+        }
+
+        * {
+            box-sizing: border-box;
+        }
     </style>
-  </head>
-  <body>
-<script type="text/javascript">
-window.CELLXGENE = {};
-window.CELLXGENE.API = {
-  prefix: window.location.href + "api/",
-  version: "v0.2/"
-};
-</script>
-    <noscript>If you're seeing this message, that means <strong>JavaScript has been disabled on your browser</strong>, please <strong>enable JS</strong> to make this app work.</noscript>
 
-    <div id="root"></div>
-  </body>
+</head>
+<body>
+<script type="text/javascript">
+  window.CELLXGENE = {};
+  window.CELLXGENE.API = {
+    prefix: window.location.href + "api/",
+    version: "v0.2/"
+  };
+</script>
+<noscript>If you're seeing this message, that means <strong>JavaScript has been disabled on your browser</strong>,
+    please <strong>enable JS</strong> to make this app work.
+</noscript>
+
+<div id="root"></div>
+{% for script in SCRIPTS %}
+<script type="text/javascript" src="{{script | safe}}"></script>
+{% endfor %}
+</body>
 </html>

client/package.json

@@ -1,20 +1,22 @@
 {
   "name": "cellxgene",
-  "version": "0.7.0",
+  "version": "0.12.0",
   "license": "MIT",
   "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
   "repository": "https://github.com/chanzuckerberg/cellxgene",
   "scripts": {
     "backend-dev": "python3.6 -m venv cellxgene && source cellxgene/bin/activate && yes | pip uninstall cellxgene || true && pip install -e .. && cellxgene launch ",
+    "backend-dev-anno": "python3.6 -m venv cellxgene && source cellxgene/bin/activate && yes | pip uninstall cellxgene || true && pip install -e .. && cellxgene launch --experimental-label-file labels.csv ",
     "build": "npm run clean && webpack --config configuration/webpack/webpack.config.prod.js",
     "clean": "rimraf build",
     "dev": "npm run clean && webpack --config configuration/webpack/webpack.config.dev.js",
-    "e2e": "jest e2e",
+    "e2e": "node node_modules/jest/bin/jest.js --verbose false --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
     "lint": "eslint src",
     "smoke-test": "start-server-and-test start-server-for-test :5000 e2e",
     "start": "node server/development.js",
     "start-server-for-test": "cellxgene launch -p 5000 ../example-dataset/pbmc3k.h5ad",
-    "unit-test": "jest --testPathIgnorePatterns e2e"
+    "test": "node node_modules/jest/bin/jest.js",
+    "unit-test": "node node_modules/jest/bin/jest.js --testPathIgnorePatterns e2e"
   },
   "engineStrict": true,
   "engines": {
@@ -30,86 +32,78 @@
     "eslint-scope": "3.7.1"
   },
   "dependencies": {
-    "@blueprintjs/core": "^3.8.0",
-    "@blueprintjs/icons": "^3.3.0",
-    "@blueprintjs/select": "^3.2.1",
-    "canvas-fit": "^1.5.0",
+    "@blueprintjs/core": "^3.18.1",
+    "@blueprintjs/icons": "^3.10.0",
+    "@blueprintjs/select": "^3.10.0",
     "d3": "^4.10.0",
-    "d3-scale-chromatic": "^1.3.0",
-    "flatbuffers": "^1.10.2",
-    "font-color-contrast": "^1.0.3",
+    "d3-scale-chromatic": "^1.5.0",
+    "flatbuffers": "^1.11.0",
     "fuzzysort": "^1.1.4",
     "gl-mat4": "^1.1.4",
-    "gl-matrix": "^2.7.1",
+    "gl-matrix": "^3.1.0",
+    "gl-vec3": "^1.1.3",
     "is-number": "^7.0.0",
-    "key-pressed": "0.0.1",
-    "lodash": "^4.17.4",
-    "memoize-one": "^4.0.0",
-    "mouse-position": "^2.0.1",
-    "mouse-pressed": "^1.0.0",
-    "normalize.css": "^8.0.0",
-    "orbit-camera": "^1.0.0",
-    "query-string": "^6.1.0",
-    "react": "^16.6.0",
+    "lodash": "^4.17.15",
+    "memoize-one": "^5.1.1",
+    "react": "^16.9.0",
     "react-autocomplete": "^1.7.2",
-    "react-dom": "^16.6.0",
-    "react-helmet": "^5.2.0",
-    "react-icons": "^3.2.2",
-    "react-redux": "^5.1.0",
-    "redux": "^4.0.1",
-    "redux-devtools-extension": "^2.13.5",
+    "react-dom": "^16.9.0",
+    "react-helmet": "^5.2.1",
+    "react-icons": "^3.7.0",
+    "react-redux": "^7.1.1",
+    "redux": "^4.0.4",
     "redux-thunk": "^2.2.0",
-    "regl": "^1.3.9",
-    "scroll-speed": "^1.0.0",
-    "urijs": "^1.19.0"
+    "regl": "^1.3.13"
   },
   "devDependencies": {
-    "@babel/core": "^7.1.5",
-    "@babel/plugin-proposal-class-properties": "^7.0.0",
-    "@babel/plugin-proposal-decorators": "^7.0.0",
-    "@babel/plugin-proposal-export-namespace-from": "^7.0.0",
-    "@babel/plugin-proposal-function-bind": "^7.0.0",
-    "@babel/plugin-transform-react-constant-elements": "^7.0.0",
-    "@babel/plugin-transform-runtime": "^7.1.0",
-    "@babel/preset-env": "^7.1.5",
+    "@babel/core": "^7.6.0",
+    "@babel/plugin-proposal-class-properties": "^7.5.5",
+    "@babel/plugin-proposal-decorators": "^7.6.0",
+    "@babel/plugin-proposal-export-namespace-from": "^7.5.2",
+    "@babel/plugin-proposal-function-bind": "^7.2.0",
+    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.4.4",
+    "@babel/plugin-proposal-optional-chaining": "^7.6.0",
+    "@babel/plugin-transform-react-constant-elements": "^7.6.0",
+    "@babel/plugin-transform-runtime": "^7.6.0",
+    "@babel/preset-env": "^7.6.0",
     "@babel/preset-react": "^7.0.0",
-    "@babel/register": "^7.0.0",
-    "@babel/runtime": "^7.1.5",
-    "babel-core": "^7.0.0-bridge.0",
-    "babel-eslint": "^10.0.1",
-    "babel-jest": "^23.6.0",
-    "babel-loader": "^8.0.0",
-    "babel-preset-modern-browsers": "^12.0.0",
+    "@babel/register": "^7.6.0",
+    "@babel/runtime": "^7.6.0",
+    "babel-eslint": "^10.0.3",
+    "babel-jest": "^24.9.0",
+    "babel-loader": "^8.0.6",
+    "babel-preset-modern-browsers": "^14.0.0",
     "chalk": "^2.4.2",
     "connect-history-api-fallback": "^1.6.0",
-    "copy-webpack-plugin": "^4.6.0",
-    "css-loader": "^1.0.1",
-    "eslint": "^5.13.0",
-    "eslint-config-airbnb": "^17.1.0",
-    "eslint-config-prettier": "^4.0.0",
-    "eslint-loader": "^2.1.2",
+    "copy-webpack-plugin": "^5.0.4",
+    "css-loader": "^3.2.0",
+    "eslint": "^6.4.0",
+    "eslint-config-airbnb": "^18.0.1",
+    "eslint-config-prettier": "^6.3.0",
+    "eslint-loader": "^3.0.0",
     "eslint-plugin-filenames": "^1.3.2",
-    "eslint-plugin-import": "^2.16.0",
-    "eslint-plugin-jest": "^22.2.2",
-    "eslint-plugin-jsx-a11y": "^6.2.1",
-    "eslint-plugin-react": "^7.12.4",
-    "express": "^4.14.0",
-    "file-loader": "^2.0.0",
+    "eslint-plugin-import": "^2.18.2",
+    "eslint-plugin-jest": "^22.17.0",
+    "eslint-plugin-jsx-a11y": "^6.2.3",
+    "eslint-plugin-react": "^7.14.3",
+    "express": "^4.17.1",
+    "file-loader": "^4.2.0",
     "html-webpack-inline-source-plugin": "0.0.10",
     "html-webpack-plugin": "^3.2.0",
-    "jest": "^24.1.0",
+    "jest": "^24.9.0",
+    "jest-puppeteer": "^4.3.0",
     "json-loader": "^0.5.4",
-    "mini-css-extract-plugin": "^0.4.1",
-    "puppeteer": "^1.12.1",
-    "rimraf": "^2.6.3",
+    "mini-css-extract-plugin": "^0.8.0",
+    "puppeteer": "^1.20.0",
+    "rimraf": "^3.0.0",
     "serve-favicon": "^2.3.0",
-    "start-server-and-test": "^1.7.11",
-    "style-loader": "^0.23.1",
+    "start-server-and-test": "^1.10.2",
+    "style-loader": "^1.0.0",
     "sw-precache-webpack-plugin": "^0.11.5",
-    "url-loader": "^1.1.0",
-    "webpack": "^4.25.1",
-    "webpack-cli": "^3.1.0",
-    "webpack-dev-middleware": "^3.1.3"
+    "url-loader": "^2.1.0",
+    "webpack": "^4.40.2",
+    "webpack-cli": "^3.3.8",
+    "webpack-dev-middleware": "^3.7.1"
   },
   "jest": {
     "testMatch": [
@@ -129,16 +123,23 @@
         ],
         "plugins": [
           "@babel/plugin-proposal-function-bind",
-          "@babel/plugin-proposal-class-properties",
           [
             "@babel/plugin-proposal-decorators",
             {
               "legacy": true
             }
           ],
+          [
+            "@babel/plugin-proposal-class-properties",
+            {
+              "loose": true
+            }
+          ],
           "@babel/plugin-proposal-export-namespace-from",
           "@babel/plugin-transform-react-constant-elements",
-          "@babel/plugin-transform-runtime"
+          "@babel/plugin-transform-runtime",
+          "@babel/plugin-proposal-optional-chaining",
+          "@babel/plugin-proposal-nullish-coalescing-operator"
         ]
       }
     }

client/src/actions/index.js

@@ -1,7 +1,7 @@
 // jshint esversion: 6
 import _ from "lodash";
 import * as globals from "../globals";
-import { Universe } from "../util/stateManager";
+import { Universe, MatrixFBS } from "../util/stateManager";
 import {
   catchErrorsWrap,
   doJsonRequest,
@@ -21,24 +21,32 @@ const doInitialDataLoad = () =>
     dispatch({ type: "initial data load start" });
 
     try {
-      const requestJson = _(["config", "schema"])
+      /*
+      Step 1 - config & schema, all JSON
+      */
+      const requestJson = ["config", "schema"]
         .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
-        .map(url => doJsonRequest(url))
-        .value();
-      const requestBinary = _([
-        "annotations/obs",
-        "annotations/var?annotation-name=name",
-        "layout/obs"
-      ])
-        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
-        .map(url => doBinaryRequest(url))
-        .value();
-
-      const results = await Promise.all(_.concat(requestJson, requestBinary));
-
+        .map(url => doJsonRequest(url));
+      const stepOneResults = await Promise.all(requestJson);
       /* set config defaults */
-      const config = { ...globals.configDefaults, ...results[0].config };
-      const [, schema, obsAnno, varAnno, obsLayout] = [...results];
+      const config = { ...globals.configDefaults, ...stepOneResults[0].config };
+      const schema = stepOneResults[1];
+
+      /*
+      Step 2 - dataframes, all binary.  NOTE: uses results of step 1.
+      */
+      /* only load names for var annotations, if possible*/
+      const varIndexName = schema?.schema?.annotations?.var?.index;
+      const varAnnotationsQuery = varIndexName
+        ? `?annotation-name=${varIndexName}`
+        : "";
+      const varAnnotationsURL = `annotations/var${varAnnotationsQuery}`;
+      const requestBinary = ["annotations/obs", varAnnotationsURL, "layout/obs"]
+        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
+        .map(url => doBinaryRequest(url));
+      const stepTwoResults = await Promise.all(requestBinary);
+      const [obsAnno, varAnno, obsLayout] = [...stepTwoResults];
+
       const universe = Universe.createUniverseFromResponse(
         config,
         schema,
@@ -65,7 +73,7 @@ Set the view (world) to current selection.   Placeholder for an async action
 which also does re-layout.
 */
 const regraph = () => (dispatch, getState) => {
-  const { universe, world, crossfilter } = getState().controls;
+  const { universe, world, crossfilter } = getState();
   dispatch({
     type: "set World to current selection",
     universe,
@@ -91,6 +99,10 @@ needs expression data.
 Transparently utilizes cached data if it is already present.
 */
 async function _doRequestExpressionData(dispatch, getState, genes) {
+  const state = getState();
+  const { universe } = state;
+  const varIndexName = universe.schema.annotations.var.index;
+
   /* helper for this function only */
   const fetchData = async geneNames => {
     const res = await fetch(
@@ -100,7 +112,7 @@ async function _doRequestExpressionData(dispatch, getState, genes) {
         body: JSON.stringify({
           filter: {
             var: {
-              annotation_value: [{ name: "name", values: geneNames }]
+              annotation_value: [{ name: varIndexName, values: geneNames }]
             }
           }
         }),
@@ -123,8 +135,6 @@ async function _doRequestExpressionData(dispatch, getState, genes) {
     return Universe.convertDataFBStoObject(universe, data);
   };
 
-  const state = getState();
-  const { universe } = state.controls;
   /* preload data already in cache */
   let expressionData = _.transform(
     genes,
@@ -164,7 +174,7 @@ function requestSingleGeneExpressionCountsForColoringPOST(gene) {
     dispatch({ type: "get single gene expression for coloring started" });
     try {
       await _doRequestExpressionData(dispatch, getState, [gene]);
-      const { world } = getState().controls;
+      const { world } = getState();
       dispatch({
         type: "color by expression",
         gene,
@@ -185,7 +195,7 @@ const requestUserDefinedGene = gene => async (dispatch, getState) => {
   dispatch({ type: "request user defined gene started" });
   try {
     await await _doRequestExpressionData(dispatch, getState, [gene]);
-    const { world } = getState().controls;
+    const { world } = getState();
 
     /* then send the success case action through */
     return dispatch({
@@ -240,7 +250,8 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
     2. get expression data for each
     */
     const state = getState();
-    const { universe } = state.controls;
+    const { universe } = state;
+    const varIndexName = universe.schema.annotations.var.index;
 
     // Legal values are null, Array or TypedArray.  Null is initial state.
     if (!set1) set1 = [];
@@ -277,7 +288,7 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
     const data = await res.json();
     // result is [ [varIdx, ...], ... ]
     const topNGenes = _.map(data, r =>
-      universe.varAnnotations.at(r[0], "name")
+      universe.varAnnotations.at(r[0], varIndexName)
     );
 
     /*
@@ -300,8 +311,11 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
 };
 
 const resetInterface = () => (dispatch, getState) => {
-  const { universe } = getState().controls;
+  const { universe } = getState();
 
+  dispatch({
+    type: "user reset start"
+  });
   dispatch({
     type: "clear all user defined genes"
   });
@@ -321,6 +335,54 @@ const resetInterface = () => (dispatch, getState) => {
   dispatch({
     type: "increment graph render counter"
   });
+  dispatch({
+    type: "user reset end"
+  });
+};
+
+const saveObsAnnotations = () => async (dispatch, getState) => {
+  const { universe } = getState();
+  const { obsAnnotations, schema } = universe;
+
+  dispatch({
+    type: "writable obs annotations - save started"
+  });
+
+  const writableAnnotations = schema.annotations.obs.columns
+    .filter(s => s.writable)
+    .map(s => s.name);
+  const df = obsAnnotations.subset(null, writableAnnotations);
+  const matrix = MatrixFBS.encodeMatrixFBS(df);
+  try {
+    const res = await fetch(
+      `${globals.API.prefix}${globals.API.version}annotations/obs`,
+      {
+        method: "PUT",
+        body: matrix,
+        headers: new Headers({
+          "Content-Type": "application/octet-stream"
+        })
+      }
+    );
+    if (res.ok) {
+      dispatch({
+        type: "writable obs annotations - save complete",
+        obsAnnotations
+      });
+    } else {
+      dispatch({
+        type: "writable obs annotations - save error",
+        message: `HTTP error ${res.status} - ${res.statusText}`,
+        res
+      });
+    }
+  } catch (error) {
+    dispatch({
+      type: "writable obs annotations - save error",
+      message: error.toString(),
+      error
+    });
+  }
 };
 
 export default {
@@ -329,5 +391,6 @@ export default {
   requestSingleGeneExpressionCountsForColoringPOST,
   requestDifferentialExpression,
   requestUserDefinedGene,
-  doInitialDataLoad
+  doInitialDataLoad,
+  saveObsAnnotations
 };

client/src/components/app.js

@@ -4,9 +4,12 @@ import Helmet from "react-helmet";
 import { connect } from "react-redux";
 
 import Container from "./framework/container";
-import LeftSideBar from "./leftsidebar";
+import LeftSideBar from "./leftSidebar";
 import Legend from "./continuousLegend";
 import Graph from "./graph/graph";
+import MenuBar from "./menubar";
+import Autosave from "./autosave";
+
 import actions from "../actions";
 
 @connect(state => ({
@@ -71,18 +74,24 @@ class App extends React.Component {
             loading cellxgene
           </div>
         ) : null}
-        <div>
-          {loading ? null : <LeftSideBar />}
+        {error ? (
           <div
             style={{
-              padding: 15,
-              width: 1440 - 410 /* but responsive */,
-              marginLeft: 350 /* but responsive */
+              position: "fixed",
+              fontWeight: 500,
+              top: window.innerHeight / 2,
+              left: window.innerWidth / 2 - 50
             }}
           >
-            {loading ? null : <Graph key={graphRenderCounter} />}
-            <Legend />
+            error loading
           </div>
+        ) : null}
+        <div>
+          {loading ? null : <LeftSideBar />}
+          {loading ? null : <MenuBar />}
+          {loading ? null : <Graph key={graphRenderCounter} />}
+          {loading ? null : <Autosave />}
+          <Legend />
         </div>
       </Container>
     );

client/src/components/autosave/index.js

@@ -0,0 +1,81 @@
+import React from "react";
+import { connect } from "react-redux";
+
+import actions from "../../actions";
+
+@connect(state => ({
+  universe: state.universe,
+  obsAnnotations: state.universe.obsAnnotations,
+  saveInProgress: state.autosave?.saveInProgress ?? false,
+  lastSavedObsAnnotations: state.autosave?.lastSavedObsAnnotations,
+  error: state.autosave?.error,
+  writableCategoriesEnabled: state.config?.parameters?.["label_file"] ?? false
+}))
+class Autosave extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = {
+      timer: null
+    };
+  }
+
+  componentDidMount() {
+    const { writableCategoriesEnabled } = this.props;
+
+    let { timer } = this.state;
+    if (timer) clearInterval(timer);
+    if (writableCategoriesEnabled) {
+      timer = setInterval(this.tick, 2500);
+    } else {
+      timer = null;
+    }
+    this.setState({ timer });
+  }
+
+  componentWillUnmount() {
+    const { timer } = this.state;
+    if (timer) this.clearInterval(timer);
+  }
+
+  tick = () => {
+    const { dispatch, saveInProgress } = this.props;
+    if (this.needToSave() && !saveInProgress) {
+      dispatch(actions.saveObsAnnotations());
+    }
+  };
+
+  needToSave = () => {
+    /* return true if we need to save, false if we don't */
+    const { obsAnnotations, lastSavedObsAnnotations } = this.props;
+    return (
+      lastSavedObsAnnotations && obsAnnotations !== lastSavedObsAnnotations
+    );
+  };
+
+  statusMessage() {
+    const { error } = this.props;
+    if (error) {
+      return `Autosave error: ${error}`;
+    }
+    return this.needToSave() ? "Unsaved" : "All saved";
+  }
+
+  render() {
+    const { writableCategoriesEnabled } = this.props;
+    return writableCategoriesEnabled ? (
+      <div
+        id="autosave"
+        style={{
+          position: "fixed",
+          display: "inherit",
+          right: 5,
+          bottom: 5
+        }}
+      >
+        {this.statusMessage()}
+      </div>
+    ) : null;
+  }
+}
+
+export default Autosave;

client/src/components/brushableHistogram/index.js

@@ -5,69 +5,71 @@ https://bl.ocks.org/SpaceActuary/2f004899ea1b2bd78d6f1dbb2febf771
 */
 // jshint esversion: 6
 import React from "react";
-import _ from "lodash";
 import { Button, ButtonGroup, Tooltip } from "@blueprintjs/core";
 import { connect } from "react-redux";
 import * as d3 from "d3";
 import memoize from "memoize-one";
 import * as globals from "../../globals";
 import actions from "../../actions";
-import finiteExtent from "../../util/finiteExtent";
+import { makeContinuousDimensionName } from "../../util/nameCreators";
 
-@connect(state => ({
-  world: state.controls.world,
-  scatterplotXXaccessor: state.controls.scatterplotXXaccessor,
-  scatterplotYYaccessor: state.controls.scatterplotYYaccessor,
-  crossfilter: state.controls.crossfilter,
-  differential: state.differential,
-  colorAccessor: state.controls.colorAccessor,
-  colorScale: state.controls.colorScale,
-  obsAnnotations: _.get(state.controls.world, "obsAnnotations", null)
-}))
-class HistogramBrush extends React.Component {
-  calcHistogramCache = memoize((obsAnnotations, field, rangeMin, rangeMax) => {
-    const { world } = this.props;
+@connect((state, ownProps) => {
+  const { isObs, isUserDefined, isDiffExp, field } = ownProps;
+  const myName = makeContinuousDimensionName(
+    { isObs, isUserDefined, isDiffExp },
+    field
+  );
+  return {
+    world: state.world,
+    scatterplotXXaccessor: state.controls.scatterplotXXaccessor,
+    scatterplotYYaccessor: state.controls.scatterplotYYaccessor,
+    continuousSelectionRange: state.continuousSelection[myName],
+    colorAccessor: state.colors.colorAccessor
+  };
+})
+class HistogramBrush extends React.PureComponent {
+  static getColumn(world, field, clipped = true) {
+    /*
+    Return the underlying Dataframe column for our field.   By default,
+    returns the clipped column.   If clipped===false, will return the
+    unclipped column.
+    */
+    const obsAnnotations = clipped
+      ? world.obsAnnotations
+      : world.unclipped.obsAnnotations;
+    const varData = clipped ? world.varData : world.unclipped.varData;
+    if (obsAnnotations.hasCol(field)) {
+      return obsAnnotations.col(field);
+    }
+    return varData.col(field);
+  }
+
+  calcHistogramCache = memoize((col, field) => {
+    /*
+     recalculate expensive stuff, notably bins, summaries, etc.
+    */
     const histogramCache = {};
+    const values = col.asArray();
+    const summary = col.summarize();
+    const { min: domainMin, max: domainMax } = summary;
+    histogramCache.x = d3
+      .scaleLinear()
+      .domain([domainMin, domainMax])
+      .range([0, this.width - this.marginRight]);
 
+    histogramCache.bins = d3
+      .histogram()
+      .domain(histogramCache.x.domain())
+      .thresholds(40)(values);
+
+    const yMax = histogramCache.bins
+      .map(b => b.length)
+      .reduce((a, b) => Math.max(a, b));
     histogramCache.y = d3
       .scaleLinear()
+      .domain([0, yMax])
       .range([this.height - this.marginBottom, 0]);
 
-    if (obsAnnotations.hasCol(field)) {
-      // recalculate expensive stuff
-      const allValuesForContinuousFieldAsArray = obsAnnotations
-        .col(field)
-        .asArray();
-
-      histogramCache.x = d3
-        .scaleLinear()
-        .domain([rangeMin, rangeMax])
-        .range([0, this.width]);
-
-      histogramCache.bins = d3
-        .histogram()
-        .domain(histogramCache.x.domain())
-        .thresholds(40)(allValuesForContinuousFieldAsArray);
-
-      histogramCache.numValues = allValuesForContinuousFieldAsArray.length;
-    } else if (world.varData.hasCol(field)) {
-      const varValues = world.varData.col(field).asArray();
-
-      histogramCache.x = d3
-        .scaleLinear()
-        .domain(
-          finiteExtent(varValues)
-        ) /* replace this if we have ranges for genes back from server like we do for annotations on cells */
-        .range([0, this.width]);
-
-      histogramCache.bins = d3
-        .histogram()
-        .domain(histogramCache.x.domain())
-        .thresholds(40)(varValues);
-
-      histogramCache.numValues = varValues.length;
-    }
-
     return histogramCache;
   });
 
@@ -76,32 +78,90 @@ class HistogramBrush extends React.Component {
 
     this.width = 340;
     this.height = 100;
-    this.marginBottom = 20;
+    this.marginBottom = 20; // space for X axis & labels
+    this.marginRight = 40; // space for Y axis & labels
   }
 
   componentDidMount() {
     const { field } = this.props;
-    const { x, y, bins, numValues, svgRef } = this._histogram;
+    const { x, y, bins, svgRef } = this._histogram;
 
-    this.renderAxesBrushBins(x, y, bins, numValues, svgRef, field);
+    this.renderAxesBrushBins(x, y, bins, svgRef, field);
   }
 
   componentDidUpdate(prevProps) {
-    const { field, obsAnnotations } = this.props;
-    const { x, y, bins, numValues, svgRef } = this._histogram;
+    const { field, world } = this.props;
+    const { x, y, bins, svgRef } = this._histogram;
+    let { brushXselection, brushX } = this.state;
+    let forceBrushUpdate = false;
 
-    if (obsAnnotations !== prevProps.obsAnnotations) {
-      this.renderAxesBrushBins(x, y, bins, numValues, svgRef, field);
+    /*
+    Update our axis if the underlying dataframe column has changed
+    */
+    const dfColumn = HistogramBrush.getColumn(world, field);
+    const oldDfColumn = HistogramBrush.getColumn(
+      prevProps.world,
+      prevProps.field
+    );
+    if (dfColumn !== oldDfColumn) {
+      ({ brushXselection, brushX } = this.renderAxesBrushBins(
+        x,
+        y,
+        bins,
+        svgRef,
+        field
+      ));
+      forceBrushUpdate = true;
+    }
+
+    /*
+    if the selection has changed, ensure that the brush correctly reflects
+    the underlying selection.
+    */
+    const { continuousSelectionRange: range } = this.props;
+    if (forceBrushUpdate || range !== prevProps.continuousSelectionRange) {
+      if (brushXselection) {
+        const selection = d3.brushSelection(brushXselection.node());
+        if (!range && selection) {
+          /* no active selection - clear brush */
+          brushXselection.call(brushX.move, null);
+        } else if (range && !selection) {
+          /* there is an active selection, but no brush - set the brush */
+          const x0 = x(range[0]);
+          const x1 = x(range[1]);
+          brushXselection.call(brushX.move, [x0, x1]);
+        } else if (range && selection) {
+          /* there is an active selection and a brush - make sure they match */
+          const moveDeltaThreshold = 1;
+          const x0 = x(range[0]);
+          const x1 = x(range[1]);
+          const dX0 = Math.abs(x0 - selection[0]);
+          const dX1 = Math.abs(x1 - selection[1]);
+          /*
+          only update the brush if it is grossly incorrect,
+          as defined by the moveDeltaThreshold
+          */
+          if (dX0 > moveDeltaThreshold || dX1 > moveDeltaThreshold) {
+            brushXselection.call(brushX.move, [x0, x1]);
+          }
+        }
+      }
     }
   }
 
-  onBrush(selection, x) {
+  onBrush(selection, x, eventType) {
+    const type = `continuous metadata histogram ${eventType}`;
     return () => {
       const { dispatch, field, isObs, isUserDefined, isDiffExp } = this.props;
 
+      // ignore programmatically generated events
+      if (!d3.event.sourceEvent) return;
+      // ignore cascading events, which are programmatically generated
+      if (d3.event.sourceEvent.sourceEvent) return;
+
       if (d3.event.selection) {
         dispatch({
-          type: "continuous metadata histogram brush",
+          type,
           selection: field,
           continuousNamespace: {
             isObs,
@@ -112,7 +172,7 @@ class HistogramBrush extends React.Component {
         });
       } else {
         dispatch({
-          type: "continuous metadata histogram brush",
+          type,
           selection: field,
           continuousNamespace: {
             isObs,
@@ -125,24 +185,73 @@ class HistogramBrush extends React.Component {
     };
   }
 
+  onBrushEnd(selection, x) {
+    return () => {
+      const { dispatch, field, isObs, isUserDefined, isDiffExp } = this.props;
+      const minAllowedBrushSize = 10;
+      const smallAmountToAvoidInfiniteLoop = 0.1;
+
+      // ignore programmatically generated events
+      if (!d3.event.sourceEvent) return;
+      // ignore cascading events, which are programmatically generated
+      if (d3.event.sourceEvent.sourceEvent) return;
+
+      if (d3.event.selection) {
+        let _range;
+
+        if (
+          d3.event.selection[1] - d3.event.selection[0] >
+          minAllowedBrushSize
+        ) {
+          _range = [x(d3.event.selection[0]), x(d3.event.selection[1])];
+        } else {
+          /* the user selected range is too small and will be hidden #587, so take control of it procedurally */
+          /* https://stackoverflow.com/questions/12354729/d3-js-limit-size-of-brush */
+
+          const procedurallyResizedBrushWidth =
+            d3.event.selection[0] +
+            minAllowedBrushSize +
+            smallAmountToAvoidInfiniteLoop; //
+
+          _range = [x(d3.event.selection[0]), x(procedurallyResizedBrushWidth)];
+        }
+
+        dispatch({
+          type: "continuous metadata histogram end",
+          selection: field,
+          continuousNamespace: {
+            isObs,
+            isUserDefined,
+            isDiffExp
+          },
+          range: _range
+        });
+      } else {
+        dispatch({
+          type: "continuous metadata histogram cancel",
+          selection: field,
+          continuousNamespace: {
+            isObs,
+            isUserDefined,
+            isDiffExp
+          }
+        });
+      }
+    };
+  }
+
   drawHistogram(svgRef) {
-    const { obsAnnotations, field, ranges } = this.props;
-    const histogramCache = this.calcHistogramCache(
-      obsAnnotations,
-      field,
-      ranges.min,
-      ranges.max
-    );
-
-    const { x, y, bins, numValues } = histogramCache;
-
-    this._histogram = { x, y, bins, numValues, svgRef };
+    const { field, world } = this.props;
+    const col = HistogramBrush.getColumn(world, field);
+    const histogramCache = this.calcHistogramCache(col, field);
+    const { x, y, bins } = histogramCache;
+    this._histogram = { x, y, bins, svgRef };
   }
 
   handleColorAction() {
-    const { obsAnnotations, dispatch, field, world, ranges } = this.props;
+    const { dispatch, field, world, ranges } = this.props;
 
-    if (obsAnnotations.hasCol(field)) {
+    if (world.obsAnnotations.hasCol(field)) {
       dispatch({
         type: "color by continuous metadata",
         colorAccessor: field,
@@ -204,83 +313,123 @@ class HistogramBrush extends React.Component {
     };
   }
 
-  renderAxesBrushBins(x, y, bins, numValues, svgRef, field) {
+  renderAxesBrushBins(x, y, bins, svgRef, field) {
+    const svg = d3.select(svgRef);
+
     /* Remove everything */
-    d3.select(svgRef)
-      .selectAll("*")
-      .remove();
+    svg.selectAll("*").remove();
 
     /* BINS */
-    d3.select(svgRef)
+    svg
       .insert("g", "*")
       .attr("fill", "#bbb")
       .selectAll("rect")
       .data(bins)
       .enter()
       .append("rect")
-      .attr("class", "bar")
       .attr("x", d => x(d.x0) + 1)
-      .attr("y", d => y(d.length / numValues))
+      .attr("y", d => y(d.length))
       .attr("width", d => Math.abs(x(d.x1) - x(d.x0) - 1))
-      .attr("height", d => y(0) - y(d.length / numValues));
+      .attr("height", d => y(0) - y(d.length));
 
     /* BRUSH */
-    d3.select(svgRef)
+    const brushX = d3
+      .brushX()
+      .extent([[0, 0], [this.width - this.marginRight, this.height]])
+      /*
+      emit start so that the Undoable history can save an undo point
+      upon drag start, and ignore the subsequent intermediate drag events.
+      */
+      .on("start", this.onBrush(field, x.invert, "start").bind(this))
+      .on("brush", this.onBrush(field, x.invert, "brush").bind(this))
+      .on("end", this.onBrushEnd(field, x.invert).bind(this));
+    const brushXselection = d3
+      .select(svgRef)
       .append("g")
       .attr("class", "brush")
-      .attr("data-testid", `${svgRef.id}-brush`)
-      .call(
-        d3
-          .brushX()
-          .on("brush", this.onBrush(field, x.invert).bind(this))
-          .on("end", this.onBrush(field, x.invert).bind(this))
-      );
+      .attr("data-testid", `${svgRef.dataset.testid}-brush`)
+      .call(brushX);
 
-    /* AXIS */
-    d3.select(svgRef)
+    /* X AXIS */
+    svg
       .append("g")
       .attr("class", "axis axis--x")
       .attr("transform", `translate(0,${this.height - this.marginBottom})`)
-      .call(d3.axisBottom(x).ticks(5));
+      .call(
+        d3
+          .axisBottom(x)
+          .ticks(5)
+          .tickFormat(d3.format(".0s"))
+      );
 
-    d3.select(svgRef)
-      .selectAll(".axis--x text")
-      .style("fill", "rgb(80,80,80)");
+    /* Y AXIS */
+    svg
+      .append("g")
+      .attr("class", "axis axis--y")
+      .attr("transform", `translate(${this.width - this.marginRight},0)`)
+      .call(
+        d3
+          .axisRight(y)
+          .ticks(3)
+          .tickFormat(d3.format(".0s"))
+      );
 
-    d3.select(svgRef)
-      .selectAll(".axis--x path")
-      .style("stroke", "rgb(230,230,230)");
+    /* axis style */
+    svg.selectAll(".axis text").style("fill", "rgb(80,80,80)");
+    svg.selectAll(".axis path").style("stroke", "rgb(230,230,230)");
+    svg.selectAll(".axis line").style("stroke", "rgb(230,230,230)");
 
-    d3.select(svgRef)
-      .selectAll(".axis--x line")
-      .style("stroke", "rgb(230,230,230)");
+    const newState = { brushX, brushXselection };
+    this.setState(newState);
+    return newState;
   }
 
   render() {
     const {
       field,
+      world,
       colorAccessor,
       isUserDefined,
       isDiffExp,
       logFoldChange,
-      pval,
       pvalAdj,
       scatterplotXXaccessor,
       scatterplotYYaccessor,
       zebra
     } = this.props;
+    const fieldForId = field.replace(/\s/g, "_");
+    const {
+      min: unclippedRangeMin,
+      max: unclippedRangeMax
+    } = HistogramBrush.getColumn(world, field, false).summarize();
+    const unclippedRangeMinColor =
+      world.clipQuantiles.min === 0 ? "#bbb" : globals.blue;
+    const unclippedRangeMaxColor =
+      world.clipQuantiles.max === 1 ? "#bbb" : globals.blue;
 
     return (
       <div
-        id={`histogram_${field}`}
+        id={`histogram_${fieldForId}`}
         data-testid={`histogram-${field}`}
-        data-testclass={isDiffExp ? `histogram-diffexp` : ""}
+        data-testclass={
+          isDiffExp
+            ? "histogram-diffexp"
+            : isUserDefined
+            ? "histogram-user-gene"
+            : "histogram-continuous-metadata"
+        }
         style={{
           padding: globals.leftSidebarSectionPadding,
           backgroundColor: zebra ? globals.lightestGrey : "white"
         }}
       >
-        <div style={{ display: "flex", justifyContent: "flex-end" }}>
+        <div
+          style={{
+            display: "flex",
+            justifyContent: "flex-end",
+            paddingBottom: "8px"
+          }}
+        >
           {isDiffExp || isUserDefined ? (
             <span>
               <span
@@ -289,6 +438,7 @@ class HistogramBrush extends React.Component {
               />
               <ButtonGroup style={{ marginRight: 7 }}>
                 <Button
+                  data-testid={`plot-x-${field}`}
                   onClick={this.handleSetGeneAsScatterplotX(field).bind(this)}
                   active={scatterplotXXaccessor === field}
                   intent={scatterplotXXaccessor === field ? "primary" : "none"}
@@ -296,6 +446,7 @@ class HistogramBrush extends React.Component {
                   plot x
                 </Button>
                 <Button
+                  data-testid={`plot-y-${field}`}
                   onClick={this.handleSetGeneAsScatterplotY(field).bind(this)}
                   active={scatterplotYYaccessor === field}
                   intent={scatterplotYYaccessor === field ? "primary" : "none"}
@@ -318,11 +469,17 @@ class HistogramBrush extends React.Component {
               remove
             </Button>
           ) : null}
-          <Tooltip content="Use as color scale" position="bottom">
+          <Tooltip
+            content="Use as color scale"
+            position="bottom"
+            hoverOpenDelay={globals.tooltipHoverOpenDelay}
+          >
             <Button
               onClick={this.handleColorAction.bind(this)}
               active={colorAccessor === field}
               intent={colorAccessor === field ? "primary" : "none"}
+              data-testclass="colorby"
+              data-testid={`colorby-${field}`}
               icon="tint"
             />
           </Tooltip>
@@ -330,7 +487,9 @@ class HistogramBrush extends React.Component {
         <svg
           width={this.width}
           height={this.height}
-          id={`histogram_${field}_svg`}
+          id={`histogram_${fieldForId}_svg`}
+          data-testclass="histogram-plot"
+          data-testid={`histogram-${field}-plot`}
           ref={svgRef => {
             this.drawHistogram(svgRef);
           }}
@@ -338,10 +497,21 @@ class HistogramBrush extends React.Component {
         <div
           style={{
             display: "flex",
-            justifyContent: "center"
+            justifyContent: "space-between"
           }}
         >
-          <span style={{ fontStyle: "italic" }}>{field}</span>
+          <span style={{ color: unclippedRangeMinColor }}>
+            min {unclippedRangeMin.toPrecision(4)}
+          </span>
+          <span
+            data-testclass="brushable-histogram-field-name"
+            style={{ fontStyle: "italic" }}
+          >
+            {field}
+          </span>
+          <span style={{ color: unclippedRangeMaxColor }}>
+            max {unclippedRangeMax.toPrecision(4)}
+          </span>
         </div>
 
         {isDiffExp ? (

client/src/components/categorical/categorical.css

@@ -0,0 +1,3 @@
+:local(.value):hover {
+  background: rgba(167, 182, 194, 0.3);
+}

client/src/components/categorical/categorical.js

@@ -1,17 +1,72 @@
 // jshint esversion: 6
 import React from "react";
 import _ from "lodash";
+import {
+  Button,
+  Tooltip,
+  InputGroup,
+  Dialog,
+  Classes,
+  MenuItem
+} from "@blueprintjs/core";
+import { Select } from "@blueprintjs/select";
 import { connect } from "react-redux";
 import * as globals from "../../globals";
 import Category from "./category";
 
 @connect(state => ({
-  categoricalSelectionState: state.controls.categoricalSelectionState
+  categoricalSelection: state.categoricalSelection,
+  writableCategoriesEnabled: state.config?.parameters?.["label_file"] ?? false,
+  schema: state.world?.schema
 }))
 class Categories extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = {
+      createAnnoModeActive: false,
+      newCategoryText: "",
+      categoryToDuplicate: null
+    };
+  }
+
+  handleCreateUserAnno = () => {
+    const { dispatch } = this.props;
+    const { newCategoryText, categoryToDuplicate } = this.state;
+    dispatch({
+      type: "annotation: create category",
+      data: newCategoryText,
+      categoryToDuplicate
+    });
+    this.setState({
+      createAnnoModeActive: false,
+      categoryToDuplicate: null,
+      newCategoryText: ""
+    });
+  };
+
+  handleEnableAnnoMode = () => {
+    this.setState({ createAnnoModeActive: true });
+  };
+
+  handleDisableAnnoMode = () => {
+    this.setState({ createAnnoModeActive: false });
+  };
+
+  handleModalDuplicateCategorySelection = d => {
+    this.setState({ categoryToDuplicate: d });
+  };
+
   render() {
-    const { categoricalSelectionState } = this.props;
-    if (!categoricalSelectionState) return null;
+    const { createAnnoModeActive, categoryToDuplicate } = this.state;
+    const {
+      categoricalSelection,
+      writableCategoriesEnabled,
+      schema
+    } = this.props;
+    if (!categoricalSelection) return null;
+
+    /* all names, sorted in display order.  Will be rendered in this order */
+    const allCategoryNames = Object.keys(categoricalSelection).sort();
 
     return (
       <div
@@ -19,16 +74,103 @@ class Categories extends React.Component {
           padding: globals.leftSidebarSectionPadding
         }}
       >
-        <p
-          style={Object.assign({}, globals.leftSidebarSectionHeading, {
-            marginTop: 4
-          })}
-        >
-          Categorical Metadata
-        </p>
-        {_.map(categoricalSelectionState, (catState, catName) => (
-          <Category key={catName} metadataField={catName} />
-        ))}
+        {/* READ ONLY CATEGORICAL FIELDS */}
+        {/* this is duplicative but flat, could be abstracted */}
+        {_.map(allCategoryNames, catName =>
+          !schema.annotations.obsByName[catName].writable ? (
+            <Category
+              key={catName}
+              metadataField={catName}
+              createAnnoModeActive={createAnnoModeActive}
+              isUserAnno={false}
+            />
+          ) : null
+        )}
+        {/* WRITEABLE FIELDS */}
+        {_.map(allCategoryNames, catName =>
+          schema.annotations.obsByName[catName].writable ? (
+            <Category
+              key={catName}
+              metadataField={catName}
+              createAnnoModeActive={createAnnoModeActive}
+              isUserAnno
+            />
+          ) : null
+        )}
+        {writableCategoriesEnabled ? (
+          <div>
+            <Dialog
+              icon="tag"
+              title="Create new category"
+              isOpen={createAnnoModeActive}
+              onClose={this.handleDisableAnnoMode}
+            >
+              <form
+                onSubmit={e => {
+                  e.preventDefault();
+                  this.handleCreateUserAnno();
+                }}
+              >
+                <div className={Classes.DIALOG_BODY}>
+                  <div style={{ marginBottom: 20 }}>
+                    <p>New, unique category name:</p>
+                    <InputGroup
+                      autoFocus
+                      onChange={e =>
+                        this.setState({ newCategoryText: e.target.value })
+                      }
+                      leftIcon="tag"
+                    />
+                  </div>
+                  <p>
+                    Optionally duplicate all labels & cell assignments from
+                    existing category into new category:
+                  </p>
+                  <Select
+                    items={allCategoryNames}
+                    filterable={false}
+                    itemRenderer={(d, { handleClick }) => {
+                      return (
+                        <MenuItem onClick={handleClick} key={d} text={d} />
+                      );
+                    }}
+                    noResults={<MenuItem disabled text="No results." />}
+                    onItemSelect={d => {
+                      this.handleModalDuplicateCategorySelection(d);
+                    }}
+                  >
+                    {/* children become the popover target; render value here */}
+                    <Button
+                      text={
+                        categoryToDuplicate || "None (all cells 'unassigned')"
+                      }
+                      rightIcon="double-caret-vertical"
+                    />
+                  </Select>
+                </div>
+                <div className={Classes.DIALOG_FOOTER}>
+                  <div className={Classes.DIALOG_FOOTER_ACTIONS}>
+                    <Tooltip content="Close this dialog without creating a category.">
+                      <Button onClick={this.handleDisableAnnoMode}>
+                        Cancel
+                      </Button>
+                    </Tooltip>
+                    <Button
+                      onClick={this.handleCreateUserAnno}
+                      intent="primary"
+                      type="submit"
+                    >
+                      Create new category
+                    </Button>
+                  </div>
+                </div>
+              </form>
+            </Dialog>
+            <Button onClick={this.handleEnableAnnoMode} intent="primary">
+              Create new category
+            </Button>
+          </div>
+        ) : null}
       </div>
     );
   }

client/src/components/categorical/category.js

@@ -2,57 +2,146 @@ import React from "react";
 import _ from "lodash";
 import { connect } from "react-redux";
 import { FaChevronRight, FaChevronDown } from "react-icons/fa";
-import { Button, Tooltip } from "@blueprintjs/core";
+import {
+  Button,
+  Tooltip,
+  InputGroup,
+  Menu,
+  Dialog,
+  MenuItem,
+  Popover,
+  Classes,
+  Icon,
+  Position,
+  PopoverInteractionKind
+} from "@blueprintjs/core";
 
 import * as globals from "../../globals";
 import Value from "./value";
 import sortedCategoryValues from "./util";
 
 @connect(state => ({
-  colorAccessor: state.controls.colorAccessor,
-  categoricalSelectionState: state.controls.categoricalSelectionState
+  colorAccessor: state.colors.colorAccessor,
+  categoricalSelection: state.categoricalSelection,
+  annotations: state.annotations,
+  universe: state.universe
 }))
 class Category extends React.Component {
   constructor(props) {
     super(props);
     this.state = {
       isChecked: true,
-      isExpanded: false
+      isExpanded: false,
+      newCategoryText: "",
+      newLabelText: ""
     };
   }
 
-  componentDidUpdate() {
-    const { categoricalSelectionState, metadataField } = this.props;
-    const cat = categoricalSelectionState[metadataField];
-    const categoryCount = {
-      // total number of categories in this dimension
-      totalCatCount: cat.numCategories,
-      // number of selected options in this category
-      selectedCatCount: _.reduce(
-        cat.categorySelected,
-        (res, cond) => (cond ? res + 1 : res),
-        0
-      )
-    };
-    if (categoryCount.selectedCatCount === categoryCount.totalCatCount) {
-      /* everything is on, so not indeterminate */
-      this.checkbox.indeterminate = false;
-    } else if (categoryCount.selectedCatCount === 0) {
-      /* nothing is on, so no */
-      this.checkbox.indeterminate = false;
-    } else if (categoryCount.selectedCatCount < categoryCount.totalCatCount) {
-      /* to be explicit... */
-      this.checkbox.indeterminate = true;
+  componentDidUpdate(prevProps) {
+    const { categoricalSelection, metadataField } = this.props;
+    if (categoricalSelection !== prevProps.categoricalSelection) {
+      const cat = categoricalSelection[metadataField];
+      const categoryCount = {
+        // total number of categories in this dimension
+        totalCatCount: cat.numCategoryValues,
+        // number of selected options in this category
+        selectedCatCount: _.reduce(
+          cat.categoryValueSelected,
+          (res, cond) => (cond ? res + 1 : res),
+          0
+        )
+      };
+      if (categoryCount.selectedCatCount === categoryCount.totalCatCount) {
+        /* everything is on, so not indeterminate */
+        this.checkbox.indeterminate = false;
+        this.setState({ isChecked: true }); // eslint-disable-line react/no-did-update-set-state
+      } else if (categoryCount.selectedCatCount === 0) {
+        /* nothing is on, so no */
+        this.checkbox.indeterminate = false;
+        this.setState({ isChecked: false }); // eslint-disable-line react/no-did-update-set-state
+      } else if (categoryCount.selectedCatCount < categoryCount.totalCatCount) {
+        /* to be explicit... */
+        this.checkbox.indeterminate = true;
+        this.setState({ isChecked: false });
+      }
     }
   }
 
-  handleColorChange() {
+  activateAddNewLabelMode = () => {
+    const { dispatch, metadataField } = this.props;
+    dispatch({
+      type: "annotation: activate add new label mode",
+      data: metadataField
+    });
+  };
+
+  disableAddNewLabelMode = () => {
+    const { dispatch } = this.props;
+    dispatch({
+      type: "annotation: disable add new label mode"
+    });
+  };
+
+  handleAddNewLabelToCategory = () => {
+    const { dispatch, metadataField } = this.props;
+    const { newLabelText } = this.state;
+    /*
+    XXX TODO - temporary code generates random label string.   Remove
+    when the label creation UI is implemented.
+
+    const { newLabelText } = this.state;
+    */
+    // const newLabelText = `label${Math.random()}`;
+    dispatch({
+      type: "annotation: add new label to category",
+      metadataField,
+      newLabelText
+    });
+    this.setState({ newLabelText: "" });
+  };
+
+  activateEditCategoryMode = () => {
+    const { dispatch, metadataField } = this.props;
+    dispatch({
+      type: "annotation: activate category edit mode",
+      data: metadataField
+    });
+  };
+
+  disableEditCategoryMode = () => {
+    const { dispatch } = this.props;
+    dispatch({
+      type: "annotation: disable category edit mode"
+    });
+  };
+
+  handleEditCategory = () => {
+    const { dispatch, metadataField } = this.props;
+    const { newCategoryText } = this.state;
+
+    dispatch({
+      type: "annotation: category edited",
+      metadataField,
+      newCategoryText,
+      data: newCategoryText
+    });
+  };
+
+  handleDeleteCategory = () => {
+    const { dispatch, metadataField } = this.props;
+    dispatch({
+      type: "annotation: delete category",
+      metadataField
+    });
+  };
+
+  handleColorChange = () => {
     const { dispatch, metadataField } = this.props;
     dispatch({
       type: "color by categorical metadata",
       colorAccessor: metadataField
     });
-  }
+  };
 
   toggleAll() {
     const { dispatch, metadataField } = this.props;
@@ -83,12 +172,13 @@ class Category extends React.Component {
   }
 
   renderCategoryItems() {
-    const { categoricalSelectionState, metadataField } = this.props;
+    const { categoricalSelection, metadataField, isUserAnno } = this.props;
 
-    const cat = categoricalSelectionState[metadataField];
-    const optTuples = sortedCategoryValues([...cat.categoryIndices]);
+    const cat = categoricalSelection[metadataField];
+    const optTuples = sortedCategoryValues([...cat.categoryValueIndices]);
     return _.map(optTuples, (tuple, i) => (
       <Value
+        isUserAnno={isUserAnno}
         optTuples={optTuples}
         key={tuple[1]}
         metadataField={metadataField}
@@ -99,18 +189,24 @@ class Category extends React.Component {
   }
 
   render() {
-    const { isExpanded, isChecked } = this.state;
+    const { isExpanded, isChecked, newLabelText, newCategoryText } = this.state;
     const {
       metadataField,
       colorAccessor,
-      categoricalSelectionState
+      categoricalSelection,
+      isUserAnno,
+      annotations,
+      universe
     } = this.props;
-    const { isTruncated } = categoricalSelectionState[metadataField];
+    const { isTruncated } = categoricalSelection[metadataField];
+
     return (
       <div
         style={{
           maxWidth: globals.maxControlsWidth
         }}
+        data-testclass="category"
+        data-testid={`category-${metadataField}`}
       >
         <div
           style={{
@@ -128,6 +224,8 @@ class Category extends React.Component {
           >
             <label className="bp3-control bp3-checkbox">
               <input
+                data-testclass="category-select"
+                data-testid={`category-select-${metadataField}`}
                 onChange={this.handleToggleAllClick.bind(this)}
                 ref={el => {
                   this.checkbox = el;
@@ -139,32 +237,185 @@ class Category extends React.Component {
               <span className="bp3-control-indicator" />
               {""}
             </label>
-
             <span
+              data-testid={`category-expand-${metadataField}`}
               style={{
                 cursor: "pointer",
                 display: "inline-block"
               }}
               onClick={() => {
-                this.setState({ isExpanded: !isExpanded });
+                const editingCategory =
+                  annotations.isEditingCategoryName &&
+                  annotations.categoryBeingEdited === metadataField;
+                if (!editingCategory) {
+                  this.setState({ isExpanded: !isExpanded });
+                }
               }}
             >
-              {metadataField}
-              {isExpanded ? (
-                <FaChevronDown style={{ fontSize: 10, marginLeft: 5 }} />
+              {isUserAnno ? (
+                <Icon style={{ marginRight: 5 }} icon="tag" iconSize={16} />
+              ) : null}
+
+              {annotations.isEditingCategoryName &&
+              annotations.categoryBeingEdited === metadataField ? (
+                <form
+                  style={{ display: "inline-block" }}
+                  onSubmit={e => {
+                    e.preventDefault();
+                    this.handleEditCategory();
+                  }}
+                >
+                  <InputGroup
+                    style={{ position: "relative", top: -1 }}
+                    ref={input => {
+                      this.editableCategoryInput = input;
+                    }}
+                    small
+                    autoFocus
+                    onChange={e => {
+                      this.setState({
+                        newCategoryText: e.target.value
+                      });
+                    }}
+                    defaultValue={metadataField}
+                    rightElement={
+                      <Button
+                        minimal
+                        disabled={newCategoryText.length === 0}
+                        style={{ position: "relative", top: -1 }}
+                        type="button"
+                        icon="small-tick"
+                        data-testclass="submitCategoryNameEdit"
+                        data-testid="submitCategoryNameEdit"
+                        onClick={this.handleEditCategory}
+                      />
+                    }
+                  />
+                </form>
               ) : (
-                <FaChevronRight style={{ fontSize: 10, marginLeft: 5 }} />
+                metadataField
+              )}
+
+              {isExpanded ? (
+                <FaChevronDown
+                  data-testclass="category-expand-is-expanded"
+                  style={{ fontSize: 10, marginLeft: 5 }}
+                />
+              ) : (
+                <FaChevronRight
+                  data-testclass="category-expand-is-not-expanded"
+                  style={{ fontSize: 10, marginLeft: 5 }}
+                />
               )}
             </span>
           </div>
-          <Tooltip content="Use as color scale" position="bottom">
-            <Button
-              onClick={this.handleColorChange.bind(this)}
-              active={colorAccessor === metadataField}
-              intent={colorAccessor === metadataField ? "primary" : "none"}
-              icon={"tint"}
-            />
-          </Tooltip>
+          <div>
+            {isUserAnno ? (
+              <>
+                <Dialog
+                  icon="tag"
+                  title="Add new label"
+                  isOpen={annotations.isAddingNewLabel}
+                  onClose={this.disableAddNewLabelMode}
+                >
+                  <form
+                    onSubmit={e => {
+                      e.preventDefault();
+                      this.handleAddNewLabelToCategory();
+                    }}
+                  >
+                    <div className={Classes.DIALOG_BODY}>
+                      <div style={{ marginBottom: 20 }}>
+                        <p>New, unique label name:</p>
+                        <InputGroup
+                          autoFocus
+                          onChange={e =>
+                            this.setState({ newLabelText: e.target.value })
+                          }
+                          leftIcon="tag"
+                        />
+                      </div>
+                    </div>
+                    <div className={Classes.DIALOG_FOOTER}>
+                      <div className={Classes.DIALOG_FOOTER_ACTIONS}>
+                        <Tooltip content="Close this dialog without adding a label.">
+                          <Button onClick={this.disableAddNewLabelMode}>
+                            Cancel
+                          </Button>
+                        </Tooltip>
+                        <Button
+                          disabled={
+                            newLabelText.length === 0 ||
+                            universe.schema.annotations.obsByName[
+                              metadataField
+                            ].categories.indexOf(newLabelText) !== -1
+                          }
+                          onClick={this.handleAddNewLabelToCategory}
+                          intent="primary"
+                          type="submit"
+                        >
+                          Add new label to category
+                        </Button>
+                      </div>
+                    </div>
+                  </form>
+                </Dialog>
+                <Popover
+                  interactionKind={PopoverInteractionKind.HOVER}
+                  boundary="window"
+                  position={Position.RIGHT_TOP}
+                  content={
+                    <Menu>
+                      <MenuItem
+                        icon="tag"
+                        data-testclass="handleAddNewLabelToCategory"
+                        data-testid={`handleAddNewLabelToCategory-${metadataField}`}
+                        onClick={this.activateAddNewLabelMode}
+                        text="Add a new label to this category"
+                      />
+                      <MenuItem
+                        icon="edit"
+                        data-testclass="activateEditCategoryMode"
+                        data-testid={`activateEditCategoryMode-${metadataField}`}
+                        onClick={this.activateEditCategoryMode}
+                        text="Edit this category's name"
+                      />
+                      <MenuItem
+                        icon="delete"
+                        intent="danger"
+                        data-testclass="handleDeleteCategory"
+                        data-testid={`handleDeleteCategory-${metadataField}`}
+                        onClick={this.handleDeleteCategory}
+                        text="Delete this category, all associated labels, and remove all cell assignments"
+                      />
+                    </Menu>
+                  }
+                >
+                  <Button
+                    style={{ marginLeft: 0 }}
+                    data-testclass="seeActions"
+                    data-testid={`seeActions-${metadataField}`}
+                    icon="more"
+                    minimal
+                  />
+                </Popover>
+              </>
+            ) : null}
+            <Tooltip
+              content="Use as color scale"
+              position="bottom"
+              hoverOpenDelay={globals.tooltipHoverOpenDelay}
+            >
+              <Button
+                data-testclass="colorby"
+                data-testid={`colorby-${metadataField}`}
+                onClick={this.handleColorChange}
+                active={colorAccessor === metadataField}
+                intent={colorAccessor === metadataField ? "primary" : "none"}
+                icon="tint"
+              />
+            </Tooltip>
+          </div>
         </div>
         <div style={{ marginLeft: 26 }}>
           {isExpanded ? this.renderCategoryItems() : null}

client/src/components/categorical/occupancy.js

@@ -1,71 +1,199 @@
 // jshint esversion: 6
 import React from "react";
-import _ from "lodash";
 import { connect } from "react-redux";
 import * as d3 from "d3";
+import {
+  Popover,
+  PopoverInteractionKind,
+  Position,
+  Classes
+} from "@blueprintjs/core";
 
 @connect()
 class Occupancy extends React.Component {
+  _WIDTH = 100;
+
+  _HEIGHT = 11;
+
+  createHistogram = () => {
+    /* 
+      Knowing that colorScale is based off continous data, 
+      createHistogram fetches the continous data in relation to the cells releveant to the catagory value.
+      It then seperates that data into 50 bins for drawing the mini-histogram
+    */
+    const {
+      world,
+      metadataField,
+      colorAccessor,
+      category,
+      categoryIndex
+    } = this.props;
+
+    if (!this.canvas) return;
+
+    const groupBy = world.obsAnnotations.col(metadataField);
+
+    const col =
+      world.obsAnnotations.col(colorAccessor) ||
+      world.varData.col(colorAccessor);
+
+    const range = col.summarize();
+
+    const histogramMap = col.histogram(
+      50,
+      [range.min, range.max],
+      groupBy
+    ); /* Because the signature changes we really need different names for histogram to differentiate signatures  */
+
+    const categoryValue = category.categoryValues[categoryIndex];
+    const bins = histogramMap.has(categoryValue)
+      ? histogramMap.get(categoryValue)
+      : new Array(50).fill(0);
+
+    const xScale = d3
+      .scaleLinear()
+      .domain([0, bins.length])
+      .range([0, this._WIDTH]);
+
+    const largestBin = Math.max(...bins);
+
+    const yScale = d3
+      .scaleLinear()
+      .domain([0, largestBin])
+      .range([0, this._HEIGHT]);
+
+    const ctx = this.canvas.getContext("2d");
+
+    ctx.fillStyle = "#000";
+
+    let x;
+    let y;
+
+    const rectWidth = this._WIDTH / bins.length;
+
+    for (let i = 0, { length } = bins; i < length; i += 1) {
+      x = xScale(i);
+      y = yScale(bins[i]);
+      ctx.fillRect(x, this._HEIGHT - y, rectWidth, y);
+    }
+  };
+
+  createOccupancyStack = () => {
+    /* 
+      Knowing that the color scale is based off of catagorical data, 
+      createOccupancyStack obtains a map showing the number if cells per colored value
+      Using the colorScale a stack of colored bars is drawn representing the map
+     */
+    const {
+      world,
+      metadataField,
+      colorAccessor,
+      category,
+      categoryIndex,
+      schema,
+      colorScale
+    } = this.props;
+
+    const ctx = this.canvas?.getContext("2d");
+
+    if (!ctx) return;
+
+    const groupBy = world.obsAnnotations.col(metadataField);
+    const occupancyMap = world.obsAnnotations
+      .col(colorAccessor)
+      .histogram(groupBy);
+
+    const occupancy = occupancyMap.get(category.categoryValues[categoryIndex]);
+
+    if (occupancy && occupancy.size > 0) {
+      // not all categories have occupancy, so occupancy may be undefined.
+      const x = d3
+        .scaleLinear()
+        /* get all the keys d[1] as an array, then find the sum */
+        .domain([0, d3.sum(Array.from(occupancy.values()))])
+        .range([0, this._WIDTH]);
+      const categories =
+        schema.annotations.obsByName[colorAccessor]?.categories;
+
+      let currentOffset = 0;
+      const dfColumn = world.obsAnnotations.col(colorAccessor);
+      const categoryValues = dfColumn.summarize().categories;
+
+      let o;
+      let scaledValue;
+      let value;
+
+      for (let i = 0, { length } = categoryValues; i < length; i += 1) {
+        value = categoryValues[i];
+        o = occupancy.get(value);
+        scaledValue = x(o);
+        ctx.fillStyle = o
+          ? colorScale(categories.indexOf(value))
+          : "rgb(255,255,255)";
+        ctx.fillRect(currentOffset, 0, o ? scaledValue : 0, this._HEIGHT);
+        currentOffset += o ? scaledValue : 0;
+      }
+    }
+  };
+
   render() {
     const {
-      occupancy,
-      colorScale,
-      categoricalSelectionState,
       colorAccessor,
-      schema
+      categoricalSelection,
+      category,
+      categoryIndex
     } = this.props;
-    const width = 100;
-    const height = 11;
 
-    const categories = _.filter(schema.annotations.obs, {
-      name: colorAccessor
-    })[0].categories;
+    this.canvas?.getContext("2d").clearRect(0, 0, this._WIDTH, this._HEIGHT);
 
-    const x = d3
-      .scaleLinear()
-      /* get all the keys d[1] as an array, then find the sum */
-      .domain([0, d3.sum(Array.from(occupancy, d => d[1]))])
-      .range([0, width]);
-
-    let currentOffset = 0;
-
-    const stacks = categoricalSelectionState[colorAccessor].categoryValues.map(
-      d => {
-        const o = occupancy.get(d);
-
-        const scaledValue = x(o);
-
-        const stackItem = {
-          key: d,
-          value: o || 0,
-          rectWidth: o ? scaledValue : 0,
-          offset: currentOffset,
-          fill: o ? colorScale(categories.indexOf(d)) : "rgb(255,255,255)"
-        };
-        currentOffset += o ? scaledValue : 0;
-        return stackItem;
-      }
-    );
+    const colorByIsCatagoricalData = !!categoricalSelection[colorAccessor];
 
     return (
-      <svg
-        style={{
-          marginRight: 5,
-          width,
-          height
+      <Popover
+        interactionKind={PopoverInteractionKind.HOVER_TARGET_ONLY}
+        hoverOpenDelay={1500}
+        hoverCloseDelay={200}
+        position={Position.LEFT}
+        modifiers={{
+          preventOverflow: { enabled: false },
+          hide: { enabled: false }
         }}
+        lazy
+        usePortal
+        disabled={colorByIsCatagoricalData}
+        popoverClassName={Classes.POPOVER_CONTENT_SIZING}
       >
-        {stacks.map(d => (
-          <rect
-            key={d.key}
-            width={d.rectWidth}
-            height={height}
-            x={d.offset}
-            title={d.metadataField}
-            fill={d.fill}
-          />
-        ))}
-      </svg>
+        <canvas
+          className="bp3-popover-targer"
+          style={{
+            marginRight: 5,
+            width: this._WIDTH,
+            height: this._HEIGHT,
+            borderBottom: colorByIsCatagoricalData
+              ? ""
+              : "solid rgb(230, 230, 230) 0.25px"
+          }}
+          width={this._WIDTH}
+          height={this._HEIGHT}
+          ref={ref => {
+            this.canvas = ref;
+            if (colorByIsCatagoricalData) this.createOccupancyStack();
+            else this.createHistogram();
+          }}
+        />
+        <div key="text" style={{ fontFamily: "Roboto", fontSize: "14px" }}>
+          <p style={{ margin: "0" }}>
+            This histograms shows the distribution of{" "}
+            <strong>{colorAccessor}</strong> within{" "}
+            <strong>{category.categoryValues[categoryIndex]}</strong>.
+            <br />
+            <br />
+            The x axis is the same for each histogram, while the y axis is
+            scaled to the largest bin within this histogram instead of the
+            largest bin within the whole category.
+          </p>
+        </div>
+      </Popover>
     );
   }
 }

client/src/components/categorical/util.js

@@ -5,7 +5,6 @@
 // return sorted index
 
 import isNumber from "is-number";
-import _ from "lodash";
 
 const sortedCategoryValues = values => {
   /* this sort could be memoized for perf */
@@ -13,7 +12,7 @@ const sortedCategoryValues = values => {
   const strings = [];
   const ints = [];
 
-  _.forEach(values, v => {
+  values.forEach(v => {
     if (isNumber(v[0])) {
       ints.push(v);
     } else {

client/src/components/categorical/value.js

@@ -1,54 +1,196 @@
 // jshint esversion: 6
 import { connect } from "react-redux";
 import React from "react";
-import _ from "lodash";
+
+import {
+  Button,
+  InputGroup,
+  Menu,
+  MenuItem,
+  Popover,
+  Position,
+  Icon,
+  PopoverInteractionKind
+} from "@blueprintjs/core";
 import Occupancy from "./occupancy";
-import { countCategoryValues2D } from "../../util/stateManager/worldUtil";
 import * as globals from "../../globals";
+import styles from "./categorical.css";
+import { Tooltip } from "@blueprintjs/core";
 
 @connect(state => ({
-  categoricalSelectionState: state.controls.categoricalSelectionState,
-  colorScale: state.controls.colorScale,
-  colorAccessor: state.controls.colorAccessor,
-  schema: _.get(state.controls.world, "schema", null),
-  world: state.controls.world
+  categoricalSelection: state.categoricalSelection,
+  annotations: state.annotations,
+  colorScale: state.colors.scale,
+  colorAccessor: state.colors.colorAccessor,
+  schema: state.world?.schema,
+  world: state.world
 }))
 class CategoryValue extends React.Component {
-  toggleOff() {
+  constructor(props) {
+    super(props);
+    this.state = {
+      editedLabelText: ""
+    };
+  }
+
+  handleDeleteValue = () => {
+    const {
+      dispatch,
+      metadataField,
+      categoryIndex,
+      categoricalSelection
+    } = this.props;
+    const category = categoricalSelection[metadataField];
+    const label = category.categoryValues[categoryIndex];
+    dispatch({
+      type: "annotation: delete label",
+      metadataField,
+      label
+    });
+  };
+
+  handleAddCurrentSelectionToThisLabel = () => {
+    const {
+      dispatch,
+      metadataField,
+      categoryIndex,
+      categoricalSelection
+    } = this.props;
+    const category = categoricalSelection[metadataField];
+    const label = category.categoryValues[categoryIndex];
+    dispatch({
+      type: "annotation: label current cell selection",
+      metadataField,
+      categoryIndex,
+      label
+    });
+  };
+
+  handleEditValue = () => {
+    const {
+      dispatch,
+      metadataField,
+      categoryIndex,
+      categoricalSelection
+    } = this.props;
+    const { editedLabelText } = this.state;
+    const category = categoricalSelection[metadataField];
+    const label = category.categoryValues[categoryIndex];
+    dispatch({
+      type: "annotation: label edited",
+      editedLabel: editedLabelText,
+      metadataField,
+      categoryIndex,
+      label
+    });
+    this.setState({ editedLabelText: "" });
+  };
+
+  activateEditLabelMode = () => {
+    const { dispatch, metadataField, categoryIndex } = this.props;
+    dispatch({
+      type: "annotation: activate edit label mode",
+      metadataField,
+      categoryIndex
+    });
+  };
+
+  cancelEdit = () => {
+    const { dispatch, metadataField, categoryIndex } = this.props;
+    dispatch({
+      type: "annotation: cancel edit label mode",
+      metadataField,
+      categoryIndex
+    });
+  };
+
+  toggleOff = () => {
     const { dispatch, metadataField, categoryIndex } = this.props;
     dispatch({
       type: "categorical metadata filter deselect",
       metadataField,
       categoryIndex
     });
-  }
+  };
 
-  toggleOn() {
+  shouldComponentUpdate = nextProps => {
+    /*
+    Checks to see if at least one of the following changed:
+    * world state
+    * the color accessor (what is currently being colored by)
+    * if this catagorical value's selection status has changed
+
+    If and only if true, update the component
+    */
+    const { props } = this;
+    const { metadataField, categoryIndex, categoricalSelection } = props;
+    const { categoricalSelection: newCategoricalSelection } = nextProps;
+
+    const valueSelectionChange =
+      categoricalSelection[metadataField].categoryValueSelected[
+        categoryIndex
+      ] !==
+      newCategoricalSelection[metadataField].categoryValueSelected[
+        categoryIndex
+      ];
+
+    const worldChange = props.world !== nextProps.world;
+    const colorAccessorChange = props.colorAccessor !== nextProps.colorAccessor;
+    const annotationsChange = props.annotations !== nextProps.annotations;
+
+    return (
+      valueSelectionChange ||
+      worldChange ||
+      colorAccessorChange ||
+      annotationsChange
+    );
+  };
+
+  toggleOn = () => {
     const { dispatch, metadataField, categoryIndex } = this.props;
     dispatch({
       type: "categorical metadata filter select",
       metadataField,
       categoryIndex
     });
-  }
+  };
+
+  handleMouseEnter = () => {
+    const { dispatch, metadataField, categoryIndex } = this.props;
+    dispatch({
+      type: "category value mouse hover start",
+      metadataField,
+      categoryIndex
+    });
+  };
+
+  handleMouseExit = () => {
+    const { dispatch, metadataField, categoryIndex } = this.props;
+    dispatch({
+      type: "category value mouse hover end",
+      metadataField,
+      categoryIndex
+    });
+  };
 
   render() {
     const {
-      categoricalSelectionState,
+      categoricalSelection,
       metadataField,
       categoryIndex,
       colorAccessor,
       colorScale,
       i,
       schema,
-      world
+      isUserAnno,
+      annotations
     } = this.props;
 
-    if (!categoricalSelectionState) return null;
+    if (!categoricalSelection) return null;
 
-    const category = categoricalSelectionState[metadataField];
-    const selected = category.categorySelected[categoryIndex];
-    const count = category.categoryCounts[categoryIndex];
+    const category = categoricalSelection[metadataField];
+    const selected = category.categoryValueSelected[categoryIndex];
+    const count = category.categoryValueCounts[categoryIndex];
     const value = category.categoryValues[categoryIndex];
     const displayString = String(
       category.categoryValues[categoryIndex]
@@ -57,34 +199,50 @@ class CategoryValue extends React.Component {
     /* this is the color scale, so add swatches below */
     const isColorBy = metadataField === colorAccessor;
     let categories = null;
-    let occupancy = null;
 
     if (isColorBy && schema) {
-      categories = _.filter(schema.annotations.obs, {
-        name: colorAccessor
-      })[0].categories;
+      categories = schema.annotations.obsByName[colorAccessor]?.categories;
     }
 
+    let truncatedString = null;
+
     if (
       colorAccessor &&
       !isColorBy &&
-      categoricalSelectionState[colorAccessor]
+      displayString.length > globals.categoryLabelDisplayStringShortLength
     ) {
-      occupancy = countCategoryValues2D(
-        metadataField,
-        colorAccessor,
-        world.obsAnnotations
-      );
+      truncatedString = `${displayString.slice(
+        0,
+        globals.categoryLabelDisplayStringShortLength / 2
+      )}…${displayString.slice(
+        -globals.categoryLabelDisplayStringShortLength / 2
+      )}`;
+    } else if (
+      displayString.length > globals.categoryLabelDisplayStringLongLength
+    ) {
+      truncatedString = `${displayString.slice(
+        0,
+        globals.categoryLabelDisplayStringLongLength / 2
+      )}…${displayString.slice(
+        -globals.categoryLabelDisplayStringLongLength / 2
+      )}`;
     }
 
     return (
       <div
         key={i}
+        className={styles.value}
+        data-testclass="categorical-row"
         style={{
+          padding: "4px 7px",
           display: "flex",
           alignItems: "baseline",
-          justifyContent: "space-between"
+          justifyContent: "space-between",
+          marginBottom: "2px",
+          borderRadius: "2px"
         }}
+        onMouseEnter={this.handleMouseEnter}
+        onMouseLeave={this.handleMouseExit}
       >
         <div
           style={{
@@ -96,33 +254,127 @@ class CategoryValue extends React.Component {
             justifyContent: "space-between"
           }}
         >
-          <label className="bp3-control bp3-checkbox">
-            <input
-              onChange={
-                selected ? this.toggleOff.bind(this) : this.toggleOn.bind(this)
-              }
-              checked={selected}
-              type="checkbox"
-            />
-            <span className="bp3-control-indicator" />
-            {displayString}
-          </label>
-          <span style={{ flexShrink: 0 }}>
-            {colorAccessor &&
-            !isColorBy &&
-            categoricalSelectionState[colorAccessor] ? (
-              <Occupancy
-                occupancy={occupancy.get(
-                  category.categoryValues[categoryIndex]
-                )}
-                {...this.props}
+          <div style={{ display: "flex", alignItems: "baseline" }}>
+            <label className="bp3-control bp3-checkbox" style={{ margin: 0 }}>
+              <input
+                onChange={selected ? this.toggleOff : this.toggleOn}
+                data-testclass="categorical-value-select"
+                data-testid={`categorical-value-select-${metadataField}-${displayString}`}
+                checked={selected}
+                type="checkbox"
               />
+              <span
+                className="bp3-control-indicator"
+                onMouseEnter={this.handleMouseExit}
+                onMouseLeave={this.handleMouseEnter}
+              />
+            </label>
+            <Tooltip
+              content={displayString}
+              disabled={truncatedString === null}
+              hoverOpenDelay={globals.tooltipHoverOpenDelayQuick}
+            >
+              <span
+                data-testid={`categorical-value-${metadataField}-${displayString}`}
+                data-testclass="categorical-value"
+                style={{
+                  color:
+                    displayString === globals.unassignedCategoryLabel
+                      ? "#ababab"
+                      : "black",
+                  fontStyle:
+                    displayString === globals.unassignedCategoryLabel
+                      ? "italic"
+                      : "normal",
+                  display: "inline-block",
+                  overflow: "hidden",
+                  lineHeight: "1.1em",
+                  height: "1.1em",
+                  wordBreak: "break-all",
+                  verticalAlign: "middle"
+                }}
+              >
+                {annotations.isEditingLabelName &&
+                annotations.labelEditable.category === metadataField &&
+                annotations.labelEditable.label === categoryIndex
+                  ? null
+                  : truncatedString || displayString}
+              </span>
+            </Tooltip>
+            {isUserAnno &&
+            annotations.labelEditable.category === metadataField &&
+            annotations.isEditingLabelName &&
+            annotations.labelEditable.label === categoryIndex ? (
+              <form
+                onSubmit={e => {
+                  e.preventDefault();
+                  this.handleEditValue();
+                }}
+              >
+                <InputGroup
+                  style={{ position: "relative", top: -1 }}
+                  ref={input => {
+                    this.editableInput = input;
+                  }}
+                  small
+                  autoFocus
+                  onChange={e => {
+                    this.setState({ editedLabelText: e.target.value });
+                  }}
+                  defaultValue={displayString}
+                  rightElement={
+                    <Button
+                      minimal
+                      style={{ position: "relative", top: -1 }}
+                      type="button"
+                      icon="small-tick"
+                      data-testclass="submitEdit"
+                      data-testid="submitEdit"
+                      onClick={this.handleEditValue}
+                    />
+                  }
+                />
+              </form>
+            ) : null}
+            {/*
+            CANCEL IT, WITH BUTTON, ESCAPE KEY, CLICK OUT, UNDO?
+
+            <Button
+            minimal
+            style={{ position: "relative", top: -1 }}
+            type="button"
+            icon="cross"
+            data-testclass="submitEdit"
+            data-testid="submitEdit"
+            onClick={this.cancelEdit}
+          /> */}
+          </div>
+          <span style={{ flexShrink: 0 }}>
+            {colorAccessor && !isColorBy && !annotations.isEditingLabelName ? (
+              <Occupancy category={category} {...this.props} />
             ) : null}
           </span>
         </div>
         <span>
-          <span>{count}</span>
+          <span
+            data-testclass="categorical-value-count"
+            data-testid={`categorical-value-count-${metadataField}-${displayString}`}
+            style={{
+              color:
+                displayString === globals.unassignedCategoryLabel
+                  ? "#ababab"
+                  : "black",
+              fontStyle:
+                displayString === globals.unassignedCategoryLabel
+                  ? "italic"
+                  : "auto"
+            }}
+          >
+            {count}
+          </span>
+
           <svg
+            display={isColorBy && categories ? "auto" : "none"}
             style={{
               marginLeft: 5,
               width: 11,
@@ -133,6 +385,57 @@ class CategoryValue extends React.Component {
                   : "inherit"
             }}
           />
+          {isUserAnno ? (
+            <span
+              onMouseEnter={this.handleMouseExit}
+              onMouseLeave={this.handleMouseEnter}
+            >
+              <Popover
+                interactionKind={PopoverInteractionKind.HOVER}
+                boundary="window"
+                position={Position.RIGHT_TOP}
+                content={
+                  <Menu>
+                    <MenuItem
+                      icon="plus"
+                      data-testclass="handleAddCurrentSelectionToThisLabel"
+                      data-testid={`handleAddCurrentSelectionToThisLabel-${metadataField}`}
+                      onClick={this.handleAddCurrentSelectionToThisLabel}
+                      text={`Re-label currently selected cells as ${displayString}`}
+                    />
+                    {displayString !== globals.unassignedCategoryLabel ? (
+                      <MenuItem
+                        icon="edit"
+                        text="Edit this label's name"
+                        data-testclass="handleEditValue"
+                        data-testid={`handleEditValue-${metadataField}`}
+                        onClick={this.activateEditLabelMode}
+                      />
+                    ) : null}
+                    {displayString !== globals.unassignedCategoryLabel ? (
+                      <MenuItem
+                        icon="delete"
+                        intent="danger"
+                        data-testclass="handleDeleteValue"
+                        data-testid={`handleDeleteValue-${metadataField}`}
+                        onClick={this.handleDeleteValue}
+                        text="Delete this label, and reassign all cells to type 'unknown'"
+                      />
+                    ) : null}
+                  </Menu>
+                }
+              >
+                <Button
+                  style={{ marginLeft: 0, position: "relative", top: -1 }}
+                  data-testclass="seeActions"
+                  data-testid={`seeActions-${metadataField}`}
+                  icon="more"
+                  small
+                  minimal
+                />
+              </Popover>
+            </span>
+          ) : null}
         </span>
       </div>
     );

client/src/components/continuous/continuous.js

@@ -9,11 +9,10 @@ import * as globals from "../../globals";
 import HistogramBrush from "../brushableHistogram";
 
 @connect(state => ({
-  obsAnnotations: _.get(state.controls.world, "obsAnnotations", null),
-  colorAccessor: state.controls.colorAccessor,
-  colorScale: state.controls.colorScale,
-  selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null),
-  schema: _.get(state.controls.world, "schema", null)
+  obsAnnotations: state.world?.obsAnnotations,
+  colorAccessor: state.colors.colorAccessor,
+  colorScale: state.colors.scale,
+  schema: state.world?.schema
 }))
 class Continuous extends React.Component {
   constructor(props) {
@@ -49,7 +48,7 @@ class Continuous extends React.Component {
     }
 
     /* initial value for iterator to simulate index, ranges is an object */
-    let zebra = -1;
+    let zebra = 0;
 
     return (
       <div>
@@ -65,18 +64,19 @@ class Continuous extends React.Component {
         ) : null}
         {obsAnnotations
           ? _.map(obsAnnotations.colIndex.keys(), key => {
-              const summary = obsAnnotations.col(key).summarize();
               const isColorField =
                 key.includes("color") || key.includes("Color");
+              if (key === schema.annotations.obs.index || isColorField)
+                return null;
+
+              const summary = obsAnnotations.col(key).summarize();
               const nonFiniteExtent =
-                summary.min === undefined || summary.max === undefined;
-              zebra += 1;
-              if (
-                !summary.categorical &&
-                key !== "name" &&
-                !isColorField &&
-                !nonFiniteExtent
-              ) {
+                summary.min === undefined ||
+                summary.max === undefined ||
+                Number.isNaN(summary.min) ||
+                Number.isNaN(summary.max);
+              if (!summary.categorical && !nonFiniteExtent) {
+                zebra += 1;
                 return (
                   <HistogramBrush
                     key={key}

client/src/components/continuous/drawAxes.js

@@ -1,83 +0,0 @@
-// jshint esversion: 6
-import * as d3 from "d3";
-import styles from "./parallelCoordinates.css";
-import { yAxis, brushstart } from "./util";
-
-const drawAxes = (
-  svg,
-  ctx,
-  dimensions,
-  xscale,
-  height,
-  width,
-  handleBrushAction,
-  handleColorAction
-) => {
-  /*****************************************
-  ******************************************
-  Handles a brush event, toggling the display of foreground lines.
-  ******************************************
-  ******************************************/
-
-  function brush() {
-    const actives = [];
-    svg
-      .selectAll(".parcoords_axis .parcoords_brush")
-      .filter(() => d3.brushSelection(this))
-      .each(d => {
-        actives.push({
-          dimension: d,
-          extent: d3.brushSelection(this)
-        });
-      });
-    /* fire action, with selected dimensions & their values */
-    handleBrushAction(actives);
-  }
-
-  const axes = svg
-    .selectAll(".parcoords_axis")
-    .data(dimensions)
-    .enter()
-    .append("g")
-    .attr("class", `${styles.axis} parcoords_axis`)
-    .attr("transform", (d, i) => `translate(${xscale(i)})`);
-
-  axes
-    .append("g")
-    .each(d => {
-      const renderAxis =
-        "axis" in d
-          ? d.axis.scale(d.scale) // custom axis
-          : yAxis.scale(d.scale); // default axis
-      d3.select(this).call(renderAxis);
-    })
-    .append("text")
-    .on("click", d => {
-      handleColorAction(d.key);
-    })
-    .attr("class", styles.title)
-    .attr("text-anchor", "start")
-    .text(d => ("description" in d ? `${d.description}  🖌️` : `${d.key}  🖌️`));
-
-  // Add and store a brush for each axis.
-  axes
-    .append("g")
-    .attr("class", `${styles.brush} parcoords_brush`)
-    .each(d => {
-      d3.select(this).call(
-        (d.brush = d3
-          .brushY()
-          .extent([[-10, 0], [10, height]])
-          .on("start", brushstart)
-          .on("brush", brush)
-          .on("end", brush))
-      );
-    })
-    .selectAll("rect")
-    .attr("x", -8)
-    .attr("width", 16);
-
-  return axes;
-};
-
-export default drawAxes;

client/src/components/continuousLegend/index.js

@@ -2,7 +2,7 @@
 import React from "react";
 import { connect } from "react-redux";
 import * as d3 from "d3";
-import { interpolateViridis, interpolateCool } from "d3-scale-chromatic";
+import { interpolateCool } from "d3-scale-chromatic";
 
 // create continuous color legend
 // http://bl.ocks.org/syntagmatic/e8ccca52559796be775553b467593a9f
@@ -98,8 +98,8 @@ const continuous = (selectorId, colorscale, colorAccessor) => {
 };
 
 @connect(state => ({
-  colorAccessor: state.controls.colorAccessor,
-  colorScale: state.controls.colorScale,
+  colorAccessor: state.colors.colorAccessor,
+  colorScale: state.colors.scale,
   responsive: state.responsive
 }))
 class ContinuousLegend extends React.Component {
@@ -112,6 +112,7 @@ class ContinuousLegend extends React.Component {
     const { colorAccessor, responsive, colorScale } = this.props;
     if (
       prevProps.colorAccessor !== colorAccessor ||
+      prevProps.colorScale !== colorScale ||
       prevProps.responsive.height !== responsive.height ||
       prevProps.responsive.width !== responsive.width
     ) {

client/src/components/framework/custom-icons.js

@@ -0,0 +1,16 @@
+/* https://github.com/palantir/blueprint/issues/2348 */
+
+<defs>
+<clipPath id="clip0">
+<rect width="16" height="16" fill="white"/>
+</clipPath>
+</defs>
+<g clip-path="url(#clip0)">
+<rect width="16" height="16" fill="white"/>
+<path d="M1.33415 8.75877C0.939491 8.36411 0.727699 7.82249 0.749957 7.2648L0.926361 2.84501C0.967947 1.80308 1.80308 0.967947 2.84501 0.926361L7.2648 0.749958C7.82249 0.727699 8.36411 0.939492 8.75877 1.33415L14.3595 6.93485C15.1405 7.7159 15.1405 8.98223 14.3595 9.76328L9.76328 14.3595C8.98223 15.1405 7.7159 15.1405 6.93485 14.3595L1.33415 8.75877Z" fill="black"/>
+<circle cx="4.5" cy="4.5" r="1.5" fill="white"/>
+<circle cx="4.5" cy="11.5" r="3.75" stroke="white" stroke-width="0.5"/>
+<circle cx="4.5" cy="11.5" r="3.5" fill="black"/>
+<line x1="4.5" y1="10" x2="4.5" y2="13" stroke="white"/>
+<line x1="3" y1="11.5" x2="6" y2="11.5" stroke="white"/>
+</g>

client/src/components/framework/logo.js

@@ -0,0 +1,18 @@
+import React from "react";
+import * as globals from "../../globals";
+
+const Logo = props => {
+  const { size } = props;
+  return (
+    <svg width={size} height={size} viewBox="0 0 48 48" fill="none">
+      <rect width="48" height="48" fill="white" />
+      <rect width="48" height="48" fill={globals.logoColor} />
+      <rect x="19" y="19" width="22" height="22" fill="white" />
+      <rect x="24" y="24" width="12" height="12" fill={globals.logoColor} />
+      <rect x="7" y="19" width="7" height="22" fill="white" />
+      <rect x="19" y="7" width="22" height="7" fill="white" />
+    </svg>
+  );
+};
+
+export default Logo;

client/src/components/geneExpression/expressionButtons.js

@@ -1,102 +0,0 @@
-// jshint esversion: 6
-import React from "react";
-import _ from "lodash";
-import { Button, AnchorButton, Tooltip } from "@blueprintjs/core";
-import { connect } from "react-redux";
-import * as globals from "../../globals";
-import actions from "../../actions";
-import CellSetButton from "./cellSetButtons";
-
-@connect(state => ({
-  differential: state.differential,
-  world: state.controls.world,
-  crossfilter: state.controls.crossfilter,
-  selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null)
-}))
-class Expression extends React.Component {
-  constructor(props) {
-    super(props);
-    this.state = {};
-  }
-
-  computeDiffExp() {
-    const { dispatch, differential } = this.props;
-    if (differential.celllist1 && differential.celllist2) {
-      dispatch(
-        actions.requestDifferentialExpression(
-          differential.celllist1,
-          differential.celllist2
-        )
-      );
-    }
-  }
-
-  clearDifferentialExpression() {
-    const { dispatch, differential } = this.props;
-    dispatch({
-      type: "clear differential expression",
-      diffExp: differential.diffExp
-    });
-    dispatch({
-      type: "clear scatterplot"
-    });
-  }
-
-  render() {
-    const { differential } = this.props;
-    if (!differential) {
-      return null;
-    }
-    const haveBothCellSets =
-      !!differential.celllist1 && !!differential.celllist2;
-    return (
-      <div
-        style={{
-          marginRight: 10,
-          marginBottom: 10,
-          paddingLeft: globals.leftSidebarSectionPadding
-        }}
-      >
-        <CellSetButton {...this.props} eitherCellSetOneOrTwo={1} />
-        <CellSetButton {...this.props} eitherCellSetOneOrTwo={2} />
-        {!differential.diffExp ? (
-          <Tooltip
-            content="Add two cells selections, see the top 15 differentially expressed genes between them"
-            position="bottom"
-          >
-            <AnchorButton
-              style={{ marginTop: 10 }}
-              disabled={!haveBothCellSets}
-              intent="primary"
-              data-testid="diffexp-button"
-              loading={differential.loading}
-              fill
-              type="button"
-              onClick={this.computeDiffExp.bind(this)}
-            >
-              Compute Differential Expression
-            </AnchorButton>
-          </Tooltip>
-        ) : null}
-        {differential.diffExp ? (
-          <Tooltip
-            content="Remove differentially expressed gene list and clear cell selections"
-            position="bottom"
-          >
-            <Button
-              type="button"
-              fill
-              style={{ marginTop: 10 }}
-              intent="warning"
-              onClick={this.clearDifferentialExpression.bind(this)}
-            >
-              Clear Differential Expression
-            </Button>
-          </Tooltip>
-        ) : null}
-      </div>
-    );
-  }
-}
-
-export default Expression;

client/src/components/geneExpression/index.js

@@ -21,8 +21,8 @@ import {
   postUserErrorToast,
   keepAroundErrorToast
 } from "../framework/toasters";
-import ExpressionButtons from "./expressionButtons";
-import finiteExtent from "../../util/finiteExtent";
+
+import { memoize } from "../../util/dataframe/util";
 
 const renderGene = (fuzzySortResult, { handleClick, modifiers, query }) => {
   if (!modifiers.matchesPredicate) {
@@ -35,6 +35,7 @@ const renderGene = (fuzzySortResult, { handleClick, modifiers, query }) => {
     <MenuItem
       active={modifiers.active}
       disabled={modifiers.disabled}
+      data-testid={`suggest-menu-item-${geneName}`}
       // Use of annotations in this way is incorrect and dataset specific.
       // See https://github.com/chanzuckerberg/cellxgene/issues/483
       // label={gene.n_counts}
@@ -57,11 +58,11 @@ const filterGenes = (query, genes) =>
 
 @connect(state => {
   return {
-    obsAnnotations: _.get(state.controls.world, "obsAnnotations", null),
+    obsAnnotations: state.world?.obsAnnotations,
     userDefinedGenes: state.controls.userDefinedGenes,
     userDefinedGenesLoading: state.controls.userDefinedGenesLoading,
-    world: state.controls.world,
-    colorAccessor: state.controls.colorAccessor,
+    world: state.world,
+    colorAccessor: state.colors.colorAccessor,
     differential: state.differential
   };
 })
@@ -70,10 +71,24 @@ class GeneExpression extends React.Component {
     super(props);
     this.state = {
       bulkAdd: "",
-      tab: "autosuggest"
+      tab: "autosuggest",
+      activeItem: null
     };
   }
 
+  _genesToUpper = listGenes => {
+    // Has to be a Map to preserve index
+    const upperGenes = new Map();
+    for (let i = 0, { length } = listGenes; i < length; i += 1) {
+      upperGenes.set(listGenes[i].toUpperCase(), i);
+    }
+
+    return upperGenes;
+  };
+
+  // eslint-disable-next-line react/sort-comp
+  _memoGenesToUpper = memoize(this._genesToUpper, arr => arr);
+
   placeholderGeneNames() {
     /*
     return a string containing gene name suggestions for use as a user hint.
@@ -85,7 +100,8 @@ class GeneExpression extends React.Component {
     */
     const { world } = this.props;
     const { varAnnotations } = world;
-    const geneNames = varAnnotations.col("name").asArray();
+    const varIndexName = world.schema.annotations.var.index;
+    const geneNames = varAnnotations.col(varIndexName).asArray();
     if (geneNames.length > 0) {
       const placeholder = [];
       let len = geneNames.length;
@@ -107,6 +123,8 @@ class GeneExpression extends React.Component {
 
   handleClick(g) {
     const { world, dispatch, userDefinedGenes } = this.props;
+    const varIndexName = world.schema.annotations.var.index;
+    if (!g) return;
     const gene = g.target;
     if (userDefinedGenes.indexOf(gene) !== -1) {
       postUserErrorToast("That gene already exists");
@@ -114,19 +132,22 @@ class GeneExpression extends React.Component {
       postUserErrorToast(
         "That's too many genes, you can have at most 15 user defined genes"
       );
-    } else if (world.varAnnotations.col("name").indexOf(gene) === undefined) {
+    } else if (
+      world.varAnnotations.col(varIndexName).indexOf(gene) === undefined
+    ) {
       postUserErrorToast("That doesn't appear to be a valid gene name.");
     } else {
-      dispatch(actions.requestUserDefinedGene(gene));
-      dispatch({
-        type: "user defined gene",
-        data: gene
-      });
+      dispatch({ type: "single user defined gene start" });
+      dispatch(actions.requestUserDefinedGene(gene)).then(
+        () => dispatch({ type: "single user defined gene complete" }),
+        () => dispatch({ type: "single user defined gene error" })
+      );
     }
   }
 
   handleBulkAddClick() {
     const { world, dispatch, userDefinedGenes } = this.props;
+    const varIndexName = world.schema.annotations.var.index;
     const { bulkAdd } = this.state;
 
     /*
@@ -135,26 +156,42 @@ class GeneExpression extends React.Component {
     */
     if (bulkAdd !== "") {
       const genes = _.pull(_.uniq(bulkAdd.split(/[ ,]+/)), "");
+      if (genes.length === 0) {
+        return keepAroundErrorToast("Must enter a gene name.");
+      }
+      const worldGenes = world.varAnnotations.col(varIndexName).asArray();
 
-      genes.forEach(gene => {
-        if (gene.length === 0) {
-          keepAroundErrorToast("Must enter a gene name.");
-        } else if (userDefinedGenes.indexOf(gene) !== -1) {
-          keepAroundErrorToast("That gene already exists");
-        } else if (
-          world.varAnnotations.col("name").indexOf(gene) === undefined
-        ) {
-          keepAroundErrorToast(
-            `${gene} doesn't appear to be a valid gene name.`
+      // These gene lists are unique enough where memoization is useless
+      const upperGenes = this._genesToUpper(genes);
+      const upperUserDefinedGenes = this._genesToUpper(userDefinedGenes);
+
+      const upperWorldGenes = this._memoGenesToUpper(worldGenes);
+
+      dispatch({ type: "bulk user defined gene start" });
+
+      Promise.all(
+        [...upperGenes.keys()].map(upperGene => {
+          if (upperUserDefinedGenes.get(upperGene) !== undefined) {
+            return keepAroundErrorToast("That gene already exists");
+          }
+
+          const indexOfGene = upperWorldGenes.get(upperGene);
+
+          if (indexOfGene === undefined) {
+            return keepAroundErrorToast(
+              `${
+                genes[upperGenes.get(upperGene)]
+              } doesn't appear to be a valid gene name.`
+            );
+          }
+          return dispatch(
+            actions.requestUserDefinedGene(worldGenes[indexOfGene])
           );
-        } else {
-          dispatch(actions.requestUserDefinedGene(gene));
-          dispatch({
-            type: "user defined gene",
-            data: gene
-          });
-        }
-      });
+        })
+      ).then(
+        () => dispatch({ type: "bulk user defined gene complete" }),
+        () => dispatch({ type: "bulk user defined gene error" })
+      );
     }
 
     this.setState({ bulkAdd: "" });
@@ -167,24 +204,12 @@ class GeneExpression extends React.Component {
       userDefinedGenesLoading,
       differential
     } = this.props;
-
-    const { tab, bulkAdd } = this.state;
+    const varIndexName = world?.schema?.annotations?.var?.index;
+    const { tab, bulkAdd, activeItem } = this.state;
 
     return (
       <div>
-        <div
-          style={{
-            marginTop: 30
-          }}
-        >
-          <p
-            style={Object.assign({}, globals.leftSidebarSectionHeading, {
-              paddingLeft: globals.leftSidebarSectionPadding,
-              margin: 0
-            })}
-          >
-            Selected Genes
-          </p>
+        <div>
           <div
             style={{
               padding: globals.leftSidebarSectionPadding
@@ -195,16 +220,18 @@ class GeneExpression extends React.Component {
               style={{ marginRight: 5 }}
               minimal
               small
+              data-testid="tab-autosuggest"
               onClick={() => {
                 this.setState({ tab: "autosuggest" });
               }}
             >
-              Autosuggest
+              Autosuggest genes
             </Button>
             <Button
               active={tab === "bulkadd"}
               minimal
               small
+              data-testid="section-bulk-add"
               onClick={() => {
                 this.setState({ tab: "bulkadd" });
               }}
@@ -221,9 +248,9 @@ class GeneExpression extends React.Component {
               }}
             >
               <Suggest
-                closeOnSelect
-                openOnKeyDown
                 resetOnSelect
+                closeOnSelect
+                resetOnClose
                 itemDisabled={
                   userDefinedGenesLoading ? () => true : () => false
                 }
@@ -232,15 +259,17 @@ class GeneExpression extends React.Component {
                   /* this happens on 'enter' */
                   this.handleClick(g);
                 }}
+                initialContent={<MenuItem disabled text="Enter a gene…" />}
                 inputProps={{ "data-testid": "gene-search" }}
                 inputValueRenderer={g => {
                   return "";
                 }}
                 itemListPredicate={filterGenes}
+                onActiveItemChange={item => this.setState({ activeItem: item })}
                 itemRenderer={renderGene.bind(this)}
                 items={
                   world && world.varAnnotations
-                    ? world.varAnnotations.col("name").asArray()
+                    ? world.varAnnotations.col(varIndexName).asArray()
                     : ["No genes"]
                 }
                 popoverProps={{ minimal: true }}
@@ -249,8 +278,9 @@ class GeneExpression extends React.Component {
                 className="bp3-button bp3-intent-primary"
                 data-testid={"add-gene"}
                 loading={userDefinedGenesLoading}
+                onClick={() => this.handleClick(activeItem)}
               >
-                Add
+                Add gene
               </Button>
             </ControlGroup>
           ) : null}
@@ -272,6 +302,7 @@ class GeneExpression extends React.Component {
                         this.setState({ bulkAdd: e.target.value });
                       }}
                       id="text-input-bulk-add"
+                      data-testid="input-bulk-add"
                       placeholder={this.placeholderGeneNames()}
                       value={bulkAdd}
                     />
@@ -280,7 +311,7 @@ class GeneExpression extends React.Component {
                       onClick={this.handleBulkAddClick.bind(this)}
                       loading={userDefinedGenesLoading}
                     >
-                      Add
+                      Add genes
                     </Button>
                   </ControlGroup>
                 </FormGroup>
@@ -307,18 +338,9 @@ class GeneExpression extends React.Component {
             : null}
         </div>
         <div>
-          <p
-            style={Object.assign({}, globals.leftSidebarSectionHeading, {
-              marginTop: 40,
-              paddingLeft: globals.leftSidebarSectionPadding
-            })}
-          >
-            Differentially Expressed Genes
-          </p>
-          <ExpressionButtons />
           {differential.diffExp
             ? _.map(differential.diffExp, (value, index) => {
-                const name = world.varAnnotations.at(value[0], "name");
+                const name = world.varAnnotations.at(value[0], varIndexName);
                 const values = world.varData.col(name);
                 if (!values) {
                   return null;

client/src/components/graph/drawPointsRegl.js

@@ -1,44 +1,67 @@
-// jshint esversion: 6
-const mat4 = require("gl-mat4");
-
-// opacity: https://github.com/spacetx/starfish/blob/master/viz/draw/regions.js
+import { glPointFlags, glPointSize } from "../../util/glHelpers";
 
 export default function(regl) {
   return regl({
     vert: `
     precision mediump float;
+
     attribute vec2 position;
     attribute vec3 color;
-    attribute float size;
+    attribute float flag;
+
     uniform float distance;
-    uniform mat4 projection, view;
-    varying vec3 fragColor;
+    uniform mat3 projView;
+    uniform float nPoints;
+    uniform float minViewportDimension;
+
+    varying vec4 fragColor;
+
+    const float zBottom = 0.99;
+    const float zMiddle = 0.;
+    const float zTop = -1.;
+
+    // import getFlags()
+    ${glPointFlags}
+
+    // get pointSize()
+    ${glPointSize}
+
     void main() {
-      gl_PointSize = 7.0 / pow(distance, 2.5) + size;
-      gl_Position = projection * view * vec4(position.x, -position.y, 0, 1);
-      fragColor = color;
+      bool isNaN, isSelected, isHighlight;
+      getFlags(flag, isNaN, isSelected, isHighlight);
+
+      float size = pointSize(nPoints, minViewportDimension, isSelected, isHighlight);
+      gl_PointSize = size * pow(distance, 0.5);
+
+      float z = isNaN ? zBottom : (isHighlight ? zTop : zMiddle);
+      vec3 xy = projView * vec3(position, 1.);
+      gl_Position = vec4(xy.xy, z, 1.);
+
+      float alpha = isNaN ? 0.9 : 1.0;
+      fragColor = vec4(color, alpha);
     }`,
 
     frag: `
     precision mediump float;
-    varying vec3 fragColor;
+    varying vec4 fragColor;
     void main() {
       if (length(gl_PointCoord.xy - 0.5) > 0.5) {
         discard;
       }
-      gl_FragColor = vec4(fragColor, 1);
+      gl_FragColor = fragColor;
     }`,
 
     attributes: {
       position: regl.prop("position"),
       color: regl.prop("color"),
-      size: regl.prop("size")
+      flag: regl.prop("flag")
     },
 
     uniforms: {
       distance: regl.prop("distance"),
-      view: regl.prop("view"),
-      projection: ({viewportWidth, viewportHeight}) => mat4.perspective([], Math.PI / 2, viewportWidth / viewportHeight, 0.01, 1000)
+      projView: regl.prop("projView"),
+      nPoints: regl.prop("nPoints"),
+      minViewportDimension: regl.prop("minViewportDimension")
     },
 
     count: regl.prop("count"),

client/src/components/graph/setupCentroidSVG.js

@@ -0,0 +1,34 @@
+import * as d3 from "d3";
+import styles from "./graph.css";
+
+export default (responsive, graphPaddingRight, xy, text, colorBy) => {
+  const containerWidth = responsive.width - graphPaddingRight;
+
+  const svg = d3
+    .select("#graphAttachPoint")
+    .append("svg")
+    .attr("id", "centroid-container")
+    .attr("data-testid", "centroid-overlay")
+    .attr("width", containerWidth)
+    .attr("height", responsive.height)
+    .attr("class", `${styles.graphSVG}`)
+    .style("z-index", 998)
+    .style("pointer-events", "none");
+  //  TODO: Create own styles, ask Colin for an explanation on the css
+  // For now I'm going to put centroid z-index at 998 and lasso on 999
+
+  const label = svg
+    .append("g")
+    .attr("transform", `translate(${xy[0]}, ${xy[1]})`);
+
+  label
+    .append("text")
+    .attr("text-anchor", "middle")
+    .text(text)
+    .style("font-family", "Roboto Condensed")
+    .style("font-size", "18px")
+    .style("font-weight", "700")
+    .style("fill", colorBy ? "black" : "rgb(32, 178, 212)");
+
+  return svg;
+};

client/src/components/graph/setupLasso.js

@@ -3,7 +3,7 @@
 import * as d3 from "d3";
 
 const Lasso = () => {
-  const dispatch = d3.dispatch("start", "end");
+  const dispatch = d3.dispatch("start", "end", "cancel");
 
   const polygonToPath = polygon =>
     `M${polygon.map(d => d.join(",")).join("L")}`;
@@ -28,6 +28,7 @@ const Lasso = () => {
 
       lassoPath = g
         .append("path")
+        .attr("data-testid", "lasso-element")
         .attr("fill", "#0bb")
         .attr("fill-opacity", 0.1)
         .attr("stroke", "#0bb")
@@ -81,6 +82,7 @@ const Lasso = () => {
         lassoPath.remove();
         lassoPath = null;
         lassoPolygon = null;
+        dispatch.call("cancel");
       }
     };
 
@@ -114,6 +116,23 @@ const Lasso = () => {
         closePath = null;
       }
     };
+
+    lasso.move = polygon => {
+      if (polygon !== lassoPolygon || polygon.length !== lassoPolygon.length) {
+        lasso.reset();
+
+        lassoPolygon = polygon;
+        lassoPath = g
+          .append("path")
+          .attr("data-testid", "lasso-element")
+          .attr("fill", "#0bb")
+          .attr("fill-opacity", 0.1)
+          .attr("stroke", "#0bb")
+          .attr("stroke-dasharray", "3, 3");
+
+        lassoPath.attr("d", `${polygonToPath(lassoPolygon)}Z`);
+      }
+    };
   };
 
   lasso.on = (type, callback) => {

client/src/components/graph/setupSVGandBrush.js

@@ -10,42 +10,57 @@ import Lasso from "./setupLasso";
 ******************************************/
 
 export default (
-  handleBrushSelectAction,
-  handleBrushDeselectAction,
+  selectionToolType,
+  handleStartAction,
+  handleDragAction,
+  handleEndAction,
+  handleCancelAction,
   responsive,
   graphPaddingRight,
-  handleLassoStart,
-  handleLassoEnd
+  graphInteractionMode
 ) => {
   const svg = d3
     .select("#graphAttachPoint")
     .append("svg")
+    .attr("id", "tool")
     .attr("data-testid", "layout-overlay")
     .attr("width", responsive.width - graphPaddingRight)
     .attr("height", responsive.height)
-    .attr("class", `${styles.graphSVG}`);
+    .attr("class", `${styles.graphSVG}`)
+    .style("z-index", 999)
+    .style("display", graphInteractionMode === "select" ? "inherit" : "none");
 
-  const brush = d3
-    .brush()
-    .extent([[0, 0], [responsive.width - graphPaddingRight, responsive.height]])
-    .on("brush", handleBrushSelectAction)
-    .on("end", handleBrushDeselectAction);
+  if (selectionToolType === "brush") {
+    const brush = d3
+      .brush()
+      .extent([
+        [0, 0],
+        [responsive.width - graphPaddingRight, responsive.height]
+      ])
+      .on("start", handleStartAction)
+      .on("brush", handleDragAction)
+      // FYI, brush doesn't generate cancel
+      .on("end", handleEndAction);
 
-  const brushContainer = svg
-    .append("g")
-    .attr("class", "graph_brush")
-    .call(brush);
+    const brushContainer = svg
+      .append("g")
+      .attr("class", "graph_brush")
+      .call(brush);
 
-  const lassoInstance = Lasso()
-    .on("end", handleLassoEnd)
-    .on("start", handleLassoStart);
+    return { svg, container: brushContainer, tool: brush };
+  }
 
-  const lasso = svg.call(lassoInstance);
+  if (selectionToolType === "lasso") {
+    const lasso = Lasso()
+      .on("end", handleEndAction)
+      // FYI, Lasso doesn't generate drag
+      .on("start", handleStartAction)
+      .on("cancel", handleCancelAction);
 
-  return {
-    svg,
-    brushContainer,
-    brush,
-    lasso
-  };
+    const lassoContainer = svg.call(lasso);
+
+    return { svg, container: lassoContainer, tool: lasso };
+  }
+
+  throw new Error("unknown graph selection tool");
 };

client/src/components/graph/util.js

@@ -1,26 +0,0 @@
-// jshint esversion: 6
-// createExpressionsCountsMap () {
-//
-//   const CHANGE_ME_MAGIC_GENE_INDEX = 5;
-//
-//   const expressionsCountsMap = {};
-//
-//   /* currently selected gene */
-//   expressionsCountsMap.geneName = this.state.expressions.data.genes[3];
-//
-//   let maxExpressionValue = 0;
-//
-//   /* create map of expressions for every cell */
-//   this.state.expressions.data.cells.map((c) => {
-//     /* cellname = 234 */
-//     expressionsCountsMap[c.cellname] = c["e"][CHANGE_ME_MAGIC_GENE_INDEX];
-//     /* collect the maximum value as we iterate */
-//     if (c["e"][CHANGE_ME_MAGIC_GENE_INDEX] > maxExpressionValue) {
-//       maxExpressionValue = c["e"][CHANGE_ME_MAGIC_GENE_INDEX]
-//     }
-//   })
-//
-//   expressionsCountsMap.maxValue = maxExpressionValue;
-//
-//   return expressionsCountsMap;
-// }

client/src/components/leftSidebar/index.js

@@ -1,16 +1,15 @@
 // jshint esversion: 6
-import _ from "lodash";
 import React from "react";
 import { connect } from "react-redux";
-import Categorical from "./categorical/categorical";
-import Continuous from "./continuous/continuous";
-import GeneExpression from "./geneExpression";
-import * as globals from "../globals";
-import DynamicScatterplot from "./scatterplot/scatterplot";
+import Categorical from "../categorical/categorical";
+import Continuous from "../continuous/continuous";
+import GeneExpression from "../geneExpression";
+import * as globals from "../../globals";
+import DynamicScatterplot from "../scatterplot/scatterplot";
+import TopLeftLogoAndTitle from "./topLeftLogoAndTitle";
 
 @connect(state => ({
   responsive: state.responsive,
-  datasetTitle: _.get(state.config, "displayNames.dataset"),
   scatterplotXXaccessor: state.controls.scatterplotXXaccessor,
   scatterplotYYaccessor: state.controls.scatterplotYYaccessor
 }))
@@ -18,7 +17,6 @@ class LeftSideBar extends React.Component {
   render() {
     const {
       responsive,
-      datasetTitle,
       scatterplotXXaccessor,
       scatterplotYYaccessor
     } = this.props;
@@ -27,8 +25,7 @@ class LeftSideBar extends React.Component {
     this magic number should be made less fragile,
     if cellxgene logo or tabs change, this must as well
     */
-    const metadataSectionPadding = 0;
-    // scatterplotXXaccessor && scatterplotYYaccessor ? 450 : 0;
+    const logoRelatedPadding = 50;
 
     return (
       <div
@@ -36,27 +33,14 @@ class LeftSideBar extends React.Component {
           position: "fixed",
           backgroundColor: "white",
           /* x y blur spread color */
-          boxShadow: "1px 0px 6px 2px rgba(153,153,153,0.4)"
+          boxShadow: "-3px 0px 6px 2px rgba(153,153,153,0.4)"
         }}
       >
-        <p
-          data-testid="header"
-          style={{
-            position: "fixed",
-            top: globals.cellxgeneTitleTopPadding,
-            left: globals.leftSidebarWidth + globals.cellxgeneTitleLeftPadding,
-            margin: 0,
-            fontSize: globals.largestFontSize,
-            color: globals.darkerGrey,
-            width: "100%"
-          }}
-        >
-          cellxgene: {datasetTitle}
-        </p>
-
+        <TopLeftLogoAndTitle />
         <div
           style={{
-            height: responsive.height - metadataSectionPadding,
+            height: responsive.height - logoRelatedPadding,
+            marginTop: logoRelatedPadding,
             width: globals.leftSidebarWidth,
             overflowY: "auto",
             overflowX: "hidden"

client/src/components/leftSidebar/topLeftLogoAndTitle.js

@@ -0,0 +1,90 @@
+// jshint esversion: 6
+import React from "react";
+import { connect } from "react-redux";
+import * as globals from "../../globals";
+import Logo from "../framework/logo";
+
+@connect(state => ({
+  responsive: state.responsive,
+  datasetTitle: state.config?.displayNames?.dataset ?? "",
+  aboutURL: state.config?.links?.["about-dataset"],
+  scatterplotXXaccessor: state.controls.scatterplotXXaccessor,
+  scatterplotYYaccessor: state.controls.scatterplotYYaccessor
+}))
+class LeftSideBar extends React.Component {
+  render() {
+    const { datasetTitle, aboutURL } = this.props;
+
+    const paddingToAvoidScrollBar = 15;
+
+    const displayTitle =
+      datasetTitle.length > globals.datasetTitleMaxCharacterCount
+        ? `${datasetTitle.substring(
+            0,
+            Math.floor(globals.datasetTitleMaxCharacterCount / 2)
+          )}…${datasetTitle.slice(
+            -Math.floor(globals.datasetTitleMaxCharacterCount / 2)
+          )}`
+        : datasetTitle;
+
+    return (
+      <div
+        style={{
+          paddingLeft: 8,
+          paddingTop: 8,
+          width: globals.leftSidebarWidth - paddingToAvoidScrollBar,
+          position: "absolute",
+          backgroundColor: "white",
+          zIndex: 8888
+          /* x y blur spread color */
+          // boxShadow: "-5px -1px 4px 2px rgba(225,225,225,0.4)"
+        }}
+      >
+        <Logo size={30} />
+        <span
+          style={{
+            fontSize: 28,
+            position: "relative",
+            top: -6,
+            fontWeight: "bold",
+            marginLeft: 5,
+            color: globals.logoColor,
+            userSelect: "none"
+          }}
+        >
+          cell
+          <span
+            style={{
+              position: "relative",
+              top: 1,
+              fontWeight: 300,
+              fontSize: 24
+            }}
+          >
+            ×
+          </span>
+          gene
+        </span>
+        <div
+          data-testid="header"
+          style={{
+            fontSize: 14,
+            position: "relative",
+            top: -6,
+            display: "inline-block",
+            width: "190px",
+            marginLeft: "7px",
+            height: "1.1em",
+            overflow: "hidden",
+            wordBreak: "break-all"
+          }}
+          title={datasetTitle}
+        >
+          {aboutURL ? <a href={aboutURL}>{displayTitle}</a> : displayTitle}
+        </div>
+      </div>
+    );
+  }
+}
+
+export default LeftSideBar;

client/src/components/menubar/cellSetButtons.js

@@ -3,9 +3,10 @@ import React from "react";
 import { AnchorButton, Tooltip } from "@blueprintjs/core";
 import { connect } from "react-redux";
 import { World } from "../../util/stateManager";
+import { tooltipHoverOpenDelay } from "../../globals";
 
 @connect()
-class CellSetButton extends React.Component {
+class CellSetButton extends React.PureComponent {
   set() {
     const {
       differential,
@@ -32,13 +33,16 @@ class CellSetButton extends React.Component {
   render() {
     const { differential, eitherCellSetOneOrTwo } = this.props;
     const cellListName = `celllist${eitherCellSetOneOrTwo}`;
+    const cellsSelected = differential[cellListName]
+      ? differential[cellListName].length
+      : 0;
     return (
       <Tooltip
         content="Save current selection for differential expression computation"
-        position="top"
+        position="bottom"
+        hoverOpenDelay={tooltipHoverOpenDelay}
       >
         <AnchorButton
-          style={{ marginRight: 10 }}
           type="button"
           disabled={differential.diffExp}
           onClick={this.set.bind(this)}
@@ -46,9 +50,10 @@ class CellSetButton extends React.Component {
         >
           {eitherCellSetOneOrTwo}
           {": "}
-          {differential[cellListName]
-            ? `${differential[cellListName].length} cells`
-            : "0 cells"}
+          <span data-testid={`cellset-count-${eitherCellSetOneOrTwo}`}>
+            {cellsSelected}
+          </span>
+          {" cells"}
         </AnchorButton>
       </Tooltip>
     );

client/src/components/menubar/clip.js

@@ -0,0 +1,138 @@
+// jshint esversion: 6
+import React from "react";
+import {
+  Position,
+  Button,
+  Popover,
+  NumericInput,
+  Icon,
+  Tooltip
+} from "@blueprintjs/core";
+import { tooltipHoverOpenDelay } from "../../globals";
+
+function Clip(props) {
+  const {
+    pendingClipPercentiles,
+    clipPercentileMin,
+    clipPercentileMax,
+    handleClipOpening,
+    handleClipClosing,
+    handleClipCommit,
+    isClipDisabled,
+    handleClipOnKeyPress,
+    handleClipPercentileMaxValueChange,
+    handleClipPercentileMinValueChange
+  } = props;
+
+  const clipMin =
+    pendingClipPercentiles?.clipPercentileMin ?? clipPercentileMin;
+  const clipMax =
+    pendingClipPercentiles?.clipPercentileMax ?? clipPercentileMax;
+  const activeClipClass =
+    clipPercentileMin > 0 || clipPercentileMax < 100
+      ? " bp3-intent-warning"
+      : "";
+
+  return (
+    <div
+      className="bp3-button-group"
+      style={{
+        marginLeft: 10
+      }}
+    >
+      <Popover
+        target={
+          <Tooltip
+            content="Clip all continuous values to a percentile range"
+            position="bottom"
+            hoverOpenDelay={tooltipHoverOpenDelay}
+          >
+            <Button
+              type="button"
+              data-testid="visualization-settings"
+              className={`bp3-button bp3-icon-timeline-bar-chart ${activeClipClass}`}
+              style={{
+                cursor: "pointer"
+              }}
+            />
+          </Tooltip>
+        }
+        position={Position.BOTTOM_RIGHT}
+        onOpening={handleClipOpening}
+        onClosing={handleClipClosing}
+        content={
+          <div
+            style={{
+              display: "flex",
+              justifyContent: "flex-start",
+              alignItems: "flex-start",
+              flexDirection: "column",
+              padding: 10
+            }}
+          >
+            <div>Clip all continuous values to percentile range</div>
+            <div
+              style={{
+                display: "flex",
+                justifyContent: "space-between",
+                alignItems: "center",
+                paddingTop: 5,
+                paddingBottom: 5
+              }}
+            >
+              <NumericInput
+                style={{ width: 50 }}
+                data-testid="clip-min-input"
+                onValueChange={handleClipPercentileMinValueChange}
+                onKeyPress={handleClipOnKeyPress}
+                value={clipMin}
+                min={0}
+                max={100}
+                fill={false}
+                minorStepSize={null}
+                rightElement={
+                  <div style={{ padding: "4px 2px" }}>
+                    <Icon icon="percentage" intent="primary" iconSize={14} />
+                  </div>
+                }
+              />
+              <span style={{ marginRight: 5, marginLeft: 5 }}> - </span>
+              <NumericInput
+                style={{ width: 50 }}
+                data-testid="clip-max-input"
+                onValueChange={handleClipPercentileMaxValueChange}
+                onKeyPress={handleClipOnKeyPress}
+                value={clipMax}
+                min={0}
+                max={100}
+                fill={false}
+                minorStepSize={null}
+                rightElement={
+                  <div style={{ padding: "4px 2px" }}>
+                    <Icon icon="percentage" intent="primary" iconSize={14} />
+                  </div>
+                }
+              />
+              <Button
+                type="button"
+                data-testid="clip-commit"
+                className="bp3-button"
+                disabled={isClipDisabled()}
+                style={{
+                  cursor: "pointer",
+                  marginRight: 5,
+                  marginLeft: 5
+                }}
+                onClick={handleClipCommit}
+              >
+                Clip
+              </Button>
+            </div>
+          </div>
+        }
+      />
+    </div>
+  );
+}
+
+export default Clip;

client/src/components/menubar/index.js

@@ -0,0 +1,508 @@
+// jshint esversion: 6
+import React from "react";
+import { connect } from "react-redux";
+import {
+  Button,
+  AnchorButton,
+  Tooltip,
+  Popover,
+  Position,
+  RadioGroup,
+  Radio
+} from "@blueprintjs/core";
+import { World } from "../../util/stateManager";
+import actions from "../../actions";
+import CellSetButton from "./cellSetButtons";
+import InformationMenu from "./infoMenu";
+import UndoRedoReset from "./undoRedoReset";
+import Clip from "./clip";
+import {
+  tooltipHoverOpenDelay,
+  tooltipHoverOpenDelayQuick
+} from "../../globals";
+
+@connect(state => ({
+  universe: state.universe,
+  world: state.world,
+  loading: state.controls.loading,
+  crossfilter: state.crossfilter,
+  differential: state.differential,
+  resettingInterface: state.controls.resettingInterface,
+  layoutChoice: state.layoutChoice,
+  graphInteractionMode: state.controls.graphInteractionMode,
+  clipPercentileMin: Math.round(100 * (state.world?.clipQuantiles?.min ?? 0)),
+  clipPercentileMax: Math.round(100 * (state.world?.clipQuantiles?.max ?? 1)),
+  userDefinedGenes: state.controls.userDefinedGenes,
+  diffexpGenes: state.controls.diffexpGenes,
+  colorAccessor: state.colors.colorAccessor,
+  scatterplotXXaccessor: state.controls.scatterplotXXaccessor,
+  scatterplotYYaccessor: state.controls.scatterplotYYaccessor,
+  celllist1: state.differential.celllist1,
+  celllist2: state.differential.celllist2,
+  libraryVersions: state.config?.library_versions, // eslint-disable-line camelcase
+  undoDisabled: state["@@undoable/past"].length === 0,
+  redoDisabled: state["@@undoable/future"].length === 0,
+  aboutLink: state.config?.links?.["about-dataset"],
+  disableDiffexp: state.config?.parameters?.["disable-diffexp"] ?? false,
+  diffexpMayBeSlow: state.config?.parameters?.["diffexp-may-be-slow"] ?? false
+}))
+class MenuBar extends React.Component {
+  static isValidDigitKeyEvent(e) {
+    /*
+    Return true if this event is necessary to enter a percent number input.
+    Return false if not.
+
+    Returns true for events with keys: backspace, control, alt, meta, [0-9],
+    or events that don't have a key.
+    */
+    if (e.key === null) return true;
+    if (e.ctrlKey || e.altKey || e.metaKey) return true;
+
+    // concept borrowed from blueprint's numericInputUtils:
+    // keys that print a single character when pressed have a `key` name of
+    // length 1. every other key has a longer `key` name (e.g. "Backspace",
+    // "ArrowUp", "Shift"). since none of those keys can print a character
+    // to the field--and since they may have important native behaviors
+    // beyond printing a character--we don't want to disable their effects.
+    const isSingleCharKey = e.key.length === 1;
+    if (!isSingleCharKey) return true;
+
+    const key = e.key.charCodeAt(0) - 48; /* "0" */
+    return key >= 0 && key <= 9;
+  }
+
+  constructor(props) {
+    super(props);
+    this.state = {
+      pendingClipPercentiles: null
+    };
+  }
+
+  isClipDisabled = () => {
+    /*
+    return true if clip button should be disabled.
+    */
+    const { pendingClipPercentiles } = this.state;
+    const clipPercentileMin = pendingClipPercentiles?.clipPercentileMin;
+    const clipPercentileMax = pendingClipPercentiles?.clipPercentileMax;
+
+    const { world } = this.props;
+    const currentClipMin = 100 * world?.clipQuantiles?.min;
+    const currentClipMax = 100 * world?.clipQuantiles?.max;
+
+    // if you change this test, be careful with logic around
+    // comparisons between undefined / NaN handling.
+    const isDisabled =
+      !(clipPercentileMin < clipPercentileMax) ||
+      (clipPercentileMin === currentClipMin &&
+        clipPercentileMax === currentClipMax);
+
+    return isDisabled;
+  };
+
+  isResetDisabled = () => {
+    /*
+    Reset should be disabled when all of the following are true:
+      * nothing is selected in the crossfilter
+      * world EQ universe
+      * nothing is colored by
+      * there are no userDefinedGenes or diffexpGenes displayed
+      * scatterplot is not displayed
+      * nothing in cellset1 or cellset2
+      * clip percentiles are [0,100]
+    */
+    const {
+      crossfilter,
+      world,
+      universe,
+      userDefinedGenes,
+      diffexpGenes,
+      colorAccessor,
+      scatterplotXXaccessor,
+      scatterplotYYaccessor,
+      celllist1,
+      celllist2,
+      clipPercentileMin,
+      clipPercentileMax
+    } = this.props;
+
+    if (!crossfilter || !world || !universe) {
+      return false;
+    }
+    const nothingSelected = crossfilter.countSelected() === crossfilter.size();
+    const nothingColoredBy = !colorAccessor;
+    const noGenes = userDefinedGenes.length === 0 && diffexpGenes.length === 0;
+    const scatterNotDpl = !scatterplotXXaccessor || !scatterplotYYaccessor;
+    const nothingInCellsets = !celllist1 && !celllist2;
+
+    return (
+      nothingSelected &&
+      World.worldEqUniverse(world, universe) &&
+      nothingColoredBy &&
+      noGenes &&
+      scatterNotDpl &&
+      nothingInCellsets &&
+      clipPercentileMax === 100 &&
+      clipPercentileMin === 0
+    );
+  };
+
+  resetInterface = () => {
+    const { dispatch } = this.props;
+    dispatch({
+      type: "interface reset started"
+    });
+    dispatch(actions.resetInterface());
+  };
+
+  handleClipOnKeyPress = e => {
+    /*
+    allow only numbers, plus other critical keys which
+    may be required to make a number
+    */
+    if (!MenuBar.isValidDigitKeyEvent(e)) {
+      e.preventDefault();
+    }
+  };
+
+  handleClipPercentileMinValueChange = v => {
+    /*
+    Ignore anything that isn't a legit number
+    */
+    if (!Number.isFinite(v)) return;
+
+    const { pendingClipPercentiles } = this.state;
+    const clipPercentileMax = pendingClipPercentiles?.clipPercentileMax;
+
+    /*
+    clamp to [0, currentClipPercentileMax]
+    */
+    if (v <= 0) v = 0;
+    if (v > 100) v = 100;
+    const clipPercentileMin = Math.round(v); // paranoia
+    this.setState({
+      pendingClipPercentiles: { clipPercentileMin, clipPercentileMax }
+    });
+  };
+
+  handleClipPercentileMaxValueChange = v => {
+    /*
+    Ignore anything that isn't a legit number
+    */
+    if (!Number.isFinite(v)) return;
+
+    const { pendingClipPercentiles } = this.state;
+    const clipPercentileMin = pendingClipPercentiles?.clipPercentileMin;
+
+    /*
+    clamp to [0, 100]
+    */
+    if (v < 0) v = 0;
+    if (v > 100) v = 100;
+    const clipPercentileMax = Math.round(v); // paranoia
+
+    this.setState({
+      pendingClipPercentiles: { clipPercentileMin, clipPercentileMax }
+    });
+  };
+
+  handleClipCommit = () => {
+    const { dispatch } = this.props;
+    const { pendingClipPercentiles } = this.state;
+    const { clipPercentileMin, clipPercentileMax } = pendingClipPercentiles;
+    const min = clipPercentileMin / 100;
+    const max = clipPercentileMax / 100;
+    dispatch({
+      type: "set clip quantiles",
+      clipQuantiles: { min, max }
+    });
+  };
+
+  handleClipOpening = () => {
+    const { clipPercentileMin, clipPercentileMax } = this.props;
+    this.setState({
+      pendingClipPercentiles: { clipPercentileMin, clipPercentileMax }
+    });
+  };
+
+  handleClipClosing = () => {
+    this.setState({ pendingClipPercentiles: null });
+  };
+
+  handleLayoutChoiceChange = e => {
+    const { dispatch } = this.props;
+    dispatch({
+      type: "set layout choice",
+      layoutChoice: e.currentTarget.value
+    });
+  };
+
+  computeDiffExp = () => {
+    const { dispatch, differential } = this.props;
+    if (differential.celllist1 && differential.celllist2) {
+      dispatch(
+        actions.requestDifferentialExpression(
+          differential.celllist1,
+          differential.celllist2
+        )
+      );
+    }
+  };
+
+  clearDifferentialExpression = () => {
+    const { dispatch, differential } = this.props;
+    dispatch({
+      type: "clear differential expression",
+      diffExp: differential.diffExp
+    });
+    dispatch({
+      type: "clear scatterplot"
+    });
+  };
+
+  renderDiffExp() {
+    /* diffexp-related buttons may be disabled */
+    const { disableDiffexp, differential, diffexpMayBeSlow } = this.props;
+    if (disableDiffexp) return null;
+
+    const haveBothCellSets =
+      !!differential.celllist1 && !!differential.celllist2;
+
+    const tipMessage =
+      "See top 10 differentially expressed genes" +
+      (diffexpMayBeSlow
+        ? " (CAUTION: large dataset - may take longer or fail)"
+        : "");
+
+    return (
+      <div className="bp3-button-group" style={{ marginRight: 10 }}>
+        <CellSetButton {...this.props} eitherCellSetOneOrTwo={1} />
+        <CellSetButton {...this.props} eitherCellSetOneOrTwo={2} />
+        {!differential.diffExp ? (
+          <Tooltip
+            content={tipMessage}
+            position="bottom"
+            hoverOpenDelay={tooltipHoverOpenDelayQuick}
+          >
+            <AnchorButton
+              disabled={!haveBothCellSets}
+              intent="primary"
+              data-testid="diffexp-button"
+              loading={differential.loading}
+              icon="left-join"
+              fill
+              type="button"
+              onClick={this.computeDiffExp}
+            >
+              Compute Differential Expression
+            </AnchorButton>
+          </Tooltip>
+        ) : null}
+
+        {differential.diffExp ? (
+          <Tooltip
+            content="Remove differentially expressed gene list and clear cell selections"
+            position="bottom"
+            hoverOpenDelay={tooltipHoverOpenDelayQuick}
+          >
+            <Button
+              type="button"
+              fill
+              intent="warning"
+              onClick={this.clearDifferentialExpression}
+            >
+              Clear Differential Expression
+            </Button>
+          </Tooltip>
+        ) : null}
+      </div>
+    );
+  }
+
+  render() {
+    const {
+      dispatch,
+      differential,
+      crossfilter,
+      resettingInterface,
+      libraryVersions,
+      undoDisabled,
+      redoDisabled,
+      selectionTool,
+      clipPercentileMin,
+      clipPercentileMax,
+      layoutChoice,
+      graphInteractionMode,
+      aboutLink
+    } = this.props;
+    const { pendingClipPercentiles } = this.state;
+
+    // constants used to create selection tool button
+    let selectionTooltip;
+    let selectionButtonClass;
+    if (selectionTool === "brush") {
+      selectionTooltip = "Brush selection";
+      selectionButtonClass = "bp3-icon-select";
+    } else {
+      selectionTooltip = "Lasso selection";
+      selectionButtonClass = "bp3-icon-polygon-filter";
+    }
+
+    return (
+      <div
+        style={{
+          position: "fixed",
+          right: 8,
+          top: 8
+        }}
+      >
+        {this.renderDiffExp()}
+        <Tooltip
+          content="Show only metadata and cells which are currently selected"
+          position="bottom"
+          hoverOpenDelay={tooltipHoverOpenDelay}
+        >
+          <AnchorButton
+            type="button"
+            data-testid="subset-button"
+            disabled={
+              crossfilter &&
+              (crossfilter.countSelected() === 0 ||
+                crossfilter.countSelected() === crossfilter.size())
+            }
+            style={{
+              marginRight: 10
+            }}
+            onClick={() => {
+              dispatch(actions.regraph());
+              dispatch({ type: "increment graph render counter" });
+            }}
+          >
+            subset to current selection
+          </AnchorButton>
+        </Tooltip>
+        <div className="bp3-button-group">
+          <Tooltip
+            content={selectionTooltip}
+            position="bottom"
+            hoverOpenDelay={tooltipHoverOpenDelay}
+          >
+            <Button
+              type="button"
+              data-testid="mode-lasso"
+              className={`bp3-button ${selectionButtonClass}`}
+              active={graphInteractionMode === "select"}
+              onClick={() => {
+                dispatch({
+                  type: "change graph interaction mode",
+                  data: "select"
+                });
+              }}
+              style={{
+                cursor: "pointer"
+              }}
+            />
+          </Tooltip>
+          <Tooltip
+            content="Drag to pan, scroll to zoom"
+            position="bottom"
+            hoverOpenDelay={tooltipHoverOpenDelay}
+          >
+            <Button
+              type="button"
+              data-testid="mode-pan-zoom"
+              className="bp3-button bp3-icon-zoom-in"
+              active={graphInteractionMode === "zoom"}
+              onClick={() => {
+                dispatch({
+                  type: "change graph interaction mode",
+                  data: "zoom"
+                });
+              }}
+              style={{
+                cursor: "pointer"
+              }}
+            />
+          </Tooltip>
+        </div>
+        <div
+          className="bp3-button-group"
+          style={{
+            marginLeft: 10
+          }}
+        >
+          <Popover
+            target={
+              <Tooltip
+                content="Select embedding for visualization"
+                position="bottom"
+                hoverOpenDelay={tooltipHoverOpenDelay}
+              >
+                <Button
+                  type="button"
+                  data-testid="layout-choice"
+                  className="bp3-button bp3-icon-heatmap"
+                  style={{
+                    cursor: "pointer"
+                  }}
+                />
+              </Tooltip>
+            }
+            position={Position.BOTTOM_RIGHT}
+            content={
+              <div
+                style={{
+                  display: "flex",
+                  justifyContent: "flex-start",
+                  alignItems: "flex-start",
+                  flexDirection: "column",
+                  padding: 10
+                }}
+              >
+                <RadioGroup
+                  label="Embedding Choice"
+                  onChange={this.handleLayoutChoiceChange}
+                  selectedValue={layoutChoice.current}
+                >
+                  {layoutChoice.available.map(name => (
+                    <Radio label={name} value={name} key={name} />
+                  ))}
+                </RadioGroup>
+              </div>
+            }
+          />
+        </div>
+        <Clip
+          pendingClipPercentiles={pendingClipPercentiles}
+          clipPercentileMin={clipPercentileMin}
+          clipPercentileMax={clipPercentileMax}
+          handleClipOpening={this.handleClipOpening}
+          handleClipClosing={this.handleClipClosing}
+          handleClipCommit={this.handleClipCommit}
+          isClipDisabled={this.isClipDisabled}
+          handleClipOnKeyPress={this.handleClipOnKeyPress}
+          handleClipPercentileMaxValueChange={
+            this.handleClipPercentileMaxValueChange
+          }
+          handleClipPercentileMinValueChange={
+            this.handleClipPercentileMinValueChange
+          }
+        />
+        <UndoRedoReset
+          dispatch={dispatch}
+          isResetDisabled={this.isResetDisabled}
+          resetInterface={this.resetInterface}
+          resettingInterface={resettingInterface}
+          undoDisabled={undoDisabled}
+          redoDisabled={redoDisabled}
+        />
+        <InformationMenu
+          libraryVersions={libraryVersions}
+          aboutLink={aboutLink}
+        />
+      </div>
+    );
+  }
+}
+
+export default MenuBar;

client/src/components/menubar/infoMenu.js

@@ -0,0 +1,72 @@
+// jshint esversion: 6
+import React from "react";
+import { Button, Popover, Menu, MenuItem, Position } from "@blueprintjs/core";
+
+function InformationMenu(props) {
+  const { libraryVersions, aboutLink } = props;
+  return (
+    <div style={{ marginLeft: 10 }} className="bp3-button-group">
+      <Popover
+        content={
+          <Menu>
+            {aboutLink ? (
+              <MenuItem
+                href={aboutLink}
+                target="_blank"
+                icon="document-open"
+                text="About this dataset"
+              />
+            ) : (
+              ""
+            )}
+
+            <MenuItem
+              href="https://chanzuckerberg.github.io/cellxgene/faq.html"
+              target="_blank"
+              icon="help"
+              text="FAQ"
+            />
+            <MenuItem
+              href="https://join-cellxgene-users.herokuapp.com/"
+              target="_blank"
+              icon="chat"
+              text="Chat"
+            />
+            <MenuItem
+              href="https://chanzuckerberg.github.io/cellxgene/"
+              target="_blank"
+              icon="book"
+              text="Docs"
+            />
+            <MenuItem
+              href="https://github.com/chanzuckerberg/cellxgene"
+              target="_blank"
+              icon="git-branch"
+              text="Github"
+            />
+            <MenuItem
+              target="_blank"
+              text={`cellxgene v${
+                libraryVersions && libraryVersions.cellxgene
+                  ? libraryVersions.cellxgene
+                  : null
+              }`}
+            />
+            <MenuItem text="MIT License" />
+          </Menu>
+        }
+        position={Position.BOTTOM_RIGHT}
+      >
+        <Button
+          type="button"
+          className="bp3-button bp3-icon-info-sign"
+          style={{
+            cursor: "pointer"
+          }}
+        />
+      </Popover>
+    </div>
+  );
+}
+
+export default InformationMenu;

client/src/components/menubar/undoRedoReset.js

@@ -0,0 +1,74 @@
+// jshint esversion: 6
+import React from "react";
+import { AnchorButton, Tooltip } from "@blueprintjs/core";
+import { tooltipHoverOpenDelay } from "../../globals";
+
+function InformationMenu(props) {
+  const {
+    resettingInterface,
+    undoDisabled,
+    redoDisabled,
+    resetInterface,
+    isResetDisabled,
+    dispatch
+  } = props;
+  return (
+    <div style={{ marginLeft: 10 }} className="bp3-button-group">
+      <Tooltip
+        content="Undo"
+        position="bottom"
+        hoverOpenDelay={tooltipHoverOpenDelay}
+      >
+        <AnchorButton
+          type="button"
+          className="bp3-button bp3-icon-undo"
+          disabled={undoDisabled}
+          onClick={() => {
+            dispatch({ type: "@@undoable/undo" });
+          }}
+          style={{
+            cursor: "pointer"
+          }}
+        />
+      </Tooltip>
+      <Tooltip
+        content="Redo"
+        position="bottom"
+        hoverOpenDelay={tooltipHoverOpenDelay}
+      >
+        <AnchorButton
+          type="button"
+          className="bp3-button bp3-icon-redo"
+          disabled={redoDisabled}
+          onClick={() => {
+            dispatch({ type: "@@undoable/redo" });
+          }}
+          style={{
+            cursor: "pointer"
+          }}
+        />
+      </Tooltip>
+      <Tooltip
+        content="Reset cellxgene, clearing all selections"
+        position="bottom"
+        hoverOpenDelay={tooltipHoverOpenDelay}
+      >
+        <AnchorButton
+          disabled={isResetDisabled()}
+          style={{ marginLeft: 10 }}
+          type="button"
+          loading={resettingInterface}
+          intent="none"
+          icon="refresh"
+          onClick={resetInterface}
+          data-testid="reset"
+          data-testclass={`resetting-${resettingInterface}`}
+        >
+          reset
+        </AnchorButton>
+      </Tooltip>
+    </div>
+  );
+}
+
+export default InformationMenu;

client/src/components/scatterplot/drawPointsRegl.js

@@ -1,44 +1,64 @@
-// jshint esversion: 6
-const mat4 = require("gl-mat4");
-
-// opacity: https://github.com/spacetx/starfish/blob/master/viz/draw/regions.js
+import { glPointFlags, glPointSize } from "../../util/glHelpers";
 
 export default function(regl) {
   return regl({
     vert: `
     precision mediump float;
+
     attribute vec2 position;
     attribute vec3 color;
-    attribute float size;
-    uniform float distance;
-    uniform mat4 projection, view;
-    varying vec3 fragColor;
+    attribute float flag;
+
+    uniform mat3 projection;
+    uniform float nPoints;
+    uniform float minViewportDimension;
+
+    varying vec4 fragColor;
+
+    const float zBottom = 0.99;
+    const float zMiddle = 0.;
+    const float zTop = -1.;
+
+    // import getFlags()
+    ${glPointFlags}
+
+    // get pointSize()
+    ${glPointSize}
+
     void main() {
-      gl_PointSize = 7.0 / pow(distance, 2.5) + size;
-      gl_Position = projection * view * vec4(position.x, -position.y, 0, 1);
-      fragColor = color;
+      bool isNaN, isSelected, isHighlight;
+      getFlags(flag, isNaN, isSelected, isHighlight);
+
+      gl_PointSize = pointSize(nPoints, minViewportDimension, isSelected, isHighlight);
+
+      float z = isNaN ? zBottom : (isHighlight ? zTop : zMiddle);
+      vec3 xy = projection * vec3(position, 1.);
+      gl_Position = vec4(xy.xy, z, 1.);
+
+      float alpha = isNaN ? 0.9 : 1.0;
+      fragColor = vec4(color, alpha);
     }`,
 
     frag: `
     precision mediump float;
-    varying vec3 fragColor;
+    varying vec4 fragColor;
     void main() {
       if (length(gl_PointCoord.xy - 0.5) > 0.5) {
         discard;
       }
-      gl_FragColor = vec4(fragColor, 1);
+      gl_FragColor = fragColor;
     }`,
 
     attributes: {
       position: regl.prop("position"),
       color: regl.prop("color"),
-      size: regl.prop("size")
+      flag: regl.prop("flag")
     },
 
     uniforms: {
-      distance: regl.prop("distance"),
-      view: regl.prop("view"),
-      projection: () => mat4.perspective([], Math.PI / 2, 1, 0.01, 1000)
+      projection: regl.prop("projection"),
+      nPoints: regl.prop("nPoints"),
+      minViewportDimension: regl.prop("minViewportDimension")
     },
 
     count: regl.prop("count"),

client/src/components/scatterplot/scatterplot.js

@@ -1,52 +1,46 @@
-// jshint esversion: 6
-// https://bl.ocks.org/Jverma/076377dd0125b1a508621441752735fc
-// https://peterbeshai.com/scatterplot-in-d3-with-voronoi-interaction.html
-
 import React from "react";
-import _ from "lodash";
 import { connect } from "react-redux";
 import { Button, ButtonGroup } from "@blueprintjs/core";
 import _regl from "regl";
 import * as d3 from "d3";
+import { mat3 } from "gl-matrix";
+import memoize from "memoize-one";
+import { isTypedArray } from "../../util/typeHelpers";
+
 import * as globals from "../../globals";
-
-import _camera from "../../util/camera";
-
 import setupScatterplot from "./setupScatterplot";
 import styles from "./scatterplot.css";
-
 import _drawPoints from "./drawPointsRegl";
-import scaleLinear from "../../util/scaleLinear";
-
 import { margin, width, height } from "./util";
 import finiteExtent from "../../util/finiteExtent";
 
+function createProjectionTF(viewportWidth, viewportHeight) {
+  /*
+  the projection transform accounts for the screen size & other layout
+  */
+  const m = mat3.create();
+  return mat3.projection(m, viewportWidth, viewportHeight);
+}
+
 @connect(state => {
-  const {
-    world,
-    crossfilter,
-    scatterplotXXaccessor,
-    scatterplotYYaccessor
-  } = state.controls;
-  const expressionX =
-    world &&
-    scatterplotXXaccessor &&
-    world.varData.hasCol(scatterplotXXaccessor)
-      ? world.varData.col(scatterplotXXaccessor).asArray()
-      : null;
-  const expressionY =
-    world &&
-    scatterplotYYaccessor &&
-    world.varData.hasCol(scatterplotYYaccessor)
-      ? world.varData.col(scatterplotYYaccessor).asArray()
-      : null;
+  const { world, crossfilter, universe } = state;
+  const { scatterplotXXaccessor, scatterplotYYaccessor } = state.controls;
+  const expressionX = scatterplotXXaccessor
+    ? world.varData.col(scatterplotXXaccessor)?.asArray()
+    : null;
+  const expressionY = scatterplotYYaccessor
+    ? world.varData.col(scatterplotYYaccessor)?.asArray()
+    : null;
 
   return {
     world,
+    universe,
 
-    colorRGB: state.controls.colorRGB,
-    colorAccessor: state.controls.colorAccessor,
-    colorScale: state.controls.colorScale,
+    colorRGB: state.colors.rgb,
+    colorScale: state.colors.scale,
+    colorAccessor: state.colors.colorAccessor,
+
+    centroidLabel: state.centroidLabel,
 
     // Accessors are var/gene names (strings)
     scatterplotXXaccessor,
@@ -59,11 +53,79 @@ import finiteExtent from "../../util/finiteExtent";
     expressionY,
 
     crossfilter,
-    // updated whenever the crossfilter selection is updated
-    selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null)
+
+    responsive: state.responsive
   };
 })
-class Scatterplot extends React.Component {
+class Scatterplot extends React.PureComponent {
+  computePointPositions = memoize((X, Y, xScale, yScale) => {
+    const positions = new Float32Array(2 * X.length);
+    for (let i = 0, len = X.length; i < len; i += 1) {
+      positions[2 * i] = xScale(X[i]);
+      positions[2 * i + 1] = yScale(Y[i]);
+    }
+    return positions;
+  });
+
+  computePointColors = memoize(rgb => {
+    /*
+    compute webgl colors for each point
+    */
+    const colors = new Float32Array(3 * rgb.length);
+    for (let i = 0, len = rgb.length; i < len; i += 1) {
+      colors.set(rgb[i], 3 * i);
+    }
+    return colors;
+  });
+
+  computeSelectedFlags = memoize(
+    (crossfilter, flagSelected, flagUnselected) => {
+      const x = crossfilter.fillByIsSelected(
+        new Float32Array(crossfilter.size()),
+        flagSelected,
+        flagUnselected
+      );
+      return x;
+    }
+  );
+
+  computePointFlags = memoize(
+    (world, crossfilter, colorAccessor, centroidLabel) => {
+      const flagSelected = 1;
+      const flagNaN = 2;
+      const flagHighlight = 4;
+
+      const flags = this.computeSelectedFlags(
+        crossfilter,
+        flagSelected,
+        0
+      ).slice();
+
+      const { metadataField, categoryField } = centroidLabel;
+      const highlightData = metadataField
+        ? world.obsAnnotations.col(metadataField)?.asArray()
+        : null;
+      const colorByColumn = colorAccessor
+        ? world.obsAnnotations.col(colorAccessor)?.asArray() ||
+          world.varData.col(colorAccessor)?.asArray()
+        : null;
+      const colorByData =
+        colorByColumn && isTypedArray(colorByColumn) ? colorByColumn : null;
+
+      if (colorByData || highlightData) {
+        for (let i = 0, len = flags.length; i < len; i += 1) {
+          if (highlightData) {
+            flags[i] += highlightData[i] === categoryField ? flagHighlight : 0;
+          }
+          if (colorByData) {
+            flags[i] += Number.isFinite(colorByData[i]) ? 0 : flagNaN;
+          }
+        }
+      }
+      return flags;
+    }
+  );
+
   constructor(props) {
     super(props);
     this.count = 0;
@@ -71,7 +133,7 @@ class Scatterplot extends React.Component {
     this.renderCache = {
       positions: null,
       colors: null,
-      sizes: null,
+      flags: null,
       xScale: null,
       yScale: null
     };
@@ -92,39 +154,37 @@ class Scatterplot extends React.Component {
       this.renderCache = { ...this.renderCache, ...scales };
     }
 
-    const camera = _camera(this.reglCanvas, { scale: true, rotate: false });
     const regl = _regl(this.reglCanvas);
-
     const drawPoints = _drawPoints(regl);
 
+    // Create render transform
+    const projectionTF = createProjectionTF(
+      this.reglCanvas.width,
+      this.reglCanvas.height
+    );
+
     // preallocate buffers
     const pointBuffer = regl.buffer();
     const colorBuffer = regl.buffer();
-    const sizeBuffer = regl.buffer();
+    const flagBuffer = regl.buffer();
 
-    const reglRender = regl.frame(() => {
-      this.reglDraw(
-        regl,
-        drawPoints,
-        sizeBuffer,
-        colorBuffer,
-        pointBuffer,
-        camera
-      );
-      camera.tick();
-    });
-
-    this.reglRenderState = "rendering";
+    this.renderPoints(
+      regl,
+      drawPoints,
+      flagBuffer,
+      colorBuffer,
+      pointBuffer,
+      projectionTF
+    );
 
     this.setState({
       regl,
-      sizeBuffer,
+      flagBuffer,
       pointBuffer,
       colorBuffer,
       svg,
-      reglRender,
-      camera,
-      drawPoints
+      drawPoints,
+      projectionTF
     });
   }
 
@@ -137,98 +197,78 @@ class Scatterplot extends React.Component {
       expressionX,
       expressionY,
       colorRGB,
-      selectionUpdate
+      colorAccessor,
+      centroidLabel
     } = this.props;
     const {
-      reglRender,
       regl,
       pointBuffer,
       colorBuffer,
-      sizeBuffer,
+      flagBuffer,
       svg,
       drawPoints,
-      camera
+      projectionTF
     } = this.state;
 
     if (
-      scatterplotXXaccessor !== prevProps.scatterplotXXaccessor || // was CLU now FTH1 etc
-      scatterplotYYaccessor !== prevProps.scatterplotYYaccessor // was CLU now FTH1 etc
+      scatterplotXXaccessor !== prevProps.scatterplotXXaccessor ||
+      scatterplotYYaccessor !== prevProps.scatterplotYYaccessor ||
+      world !== prevProps.world // shape or clip of world changed
     ) {
       const scales = Scatterplot.setupScales(expressionX, expressionY);
       this.drawAxesSVG(scales.xScale, scales.yScale, svg);
       this.renderCache = { ...this.renderCache, ...scales };
     }
 
-    if (reglRender && this.reglRenderState === "rendering") {
-      reglRender.cancel();
-      this.reglRenderState = "paused";
-    }
-
-    if (
-      world &&
-      regl &&
-      pointBuffer &&
-      colorBuffer &&
-      sizeBuffer &&
-      expressionX &&
-      expressionY &&
-      scatterplotXXaccessor &&
-      scatterplotYYaccessor
-    ) {
+    if (world && regl) {
       const { renderCache } = this;
       const { xScale, yScale } = this.renderCache;
-      const cellCount = expressionX.length;
+      let needsRepaint = false;
 
-      // Points change when expressionX or expressionY change.
-      if (
-        !renderCache.positions ||
-        expressionX !== prevProps.expressionX ||
-        expressionY !== prevProps.expressionY
-      ) {
-        if (!renderCache.positions) {
-          renderCache.positions = new Float32Array(2 * cellCount);
-        }
-        const glScaleX = scaleLinear([0, width], [-0.95, 0.95]);
-        const glScaleY = scaleLinear([0, height], [-1, 1]);
-        for (let i = 0, { positions } = renderCache; i < cellCount; i += 1) {
-          positions[2 * i] = glScaleX(xScale(expressionX[i]));
-          positions[2 * i + 1] = glScaleY(yScale(expressionY[i]));
-        }
-        pointBuffer({ data: renderCache.positions, dimension: 2 });
-      }
-
-      // Colors for each point - change only when props.colorsRGB change.
-      if (!renderCache.colors || colorRGB !== prevProps.colorRGB) {
-        if (!renderCache.colors) {
-          renderCache.colors = new Float32Array(3 * cellCount);
-        }
-        for (let i = 0, { colors } = renderCache; i < cellCount; i += 1) {
-          colors.set(colorRGB[i], 3 * i);
-        }
-        colorBuffer({ data: renderCache.colors, dimension: 3 });
-      }
-
-      // Sizes for each point - updates are triggered only when selected
-      // obs change
-      if (!renderCache.sizes || selectionUpdate !== prevProps.selctionUpdate) {
-        if (!renderCache.sizes) {
-          renderCache.sizes = new Float32Array(cellCount);
-        }
-        crossfilter.fillByIsFiltered(renderCache.sizes, 4, 0.2);
-        sizeBuffer({ data: renderCache.sizes, dimension: 1 });
-      }
-
-      this.count = cellCount;
-
-      regl._refresh();
-      this.reglDraw(
-        regl,
-        drawPoints,
-        sizeBuffer,
-        colorBuffer,
-        pointBuffer,
-        camera
+      const newPositions = this.computePointPositions(
+        expressionX,
+        expressionY,
+        xScale,
+        yScale
       );
+      if (renderCache.positions !== newPositions) {
+        renderCache.positions = newPositions;
+        pointBuffer({ data: renderCache.positions, dimension: 2 });
+        needsRepaint = true;
+      }
+
+      /* colors for each point */
+      const newColors = this.computePointColors(colorRGB);
+      if (renderCache.colors !== newColors) {
+        renderCache.colors = newColors;
+        colorBuffer({ data: renderCache.colors, dimension: 3 });
+        needsRepaint = true;
+      }
+
+      const newFlags = this.computePointFlags(
+        world,
+        crossfilter,
+        colorAccessor,
+        centroidLabel
+      );
+      if (renderCache.flags !== newFlags) {
+        renderCache.flags = newFlags;
+        flagBuffer({ data: renderCache.flags, dimension: 1 });
+        needsRepaint = true;
+      }
+
+      this.count = expressionX.length;
+
+      if (needsRepaint) {
+        this.renderPoints(
+          regl,
+          drawPoints,
+          flagBuffer,
+          colorBuffer,
+          pointBuffer,
+          projectionTF
+        );
+      }
     }
   }
 
@@ -248,31 +288,21 @@ class Scatterplot extends React.Component {
     };
   }
 
-  reglDraw(regl, drawPoints, sizeBuffer, colorBuffer, pointBuffer, camera) {
-    regl.clear({
-      depth: 1,
-      color: [1, 1, 1, 1]
-    });
-
-    drawPoints({
-      size: sizeBuffer,
-      distance: camera.distance,
-      color: colorBuffer,
-      position: pointBuffer,
-      count: this.count,
-      view: camera.view()
-    });
-  }
-
   drawAxesSVG(xScale, yScale, svg) {
     const { scatterplotYYaccessor, scatterplotXXaccessor } = this.props;
     svg.selectAll("*").remove();
 
     // the axes are much cleaner and easier now. No need to rotate and orient
     // the axis, just call axisBottom, axisLeft etc.
-    const xAxis = d3.axisBottom().scale(xScale);
+    const xAxis = d3
+      .axisBottom()
+      .ticks(7)
+      .scale(xScale);
 
-    const yAxis = d3.axisLeft().scale(yScale);
+    const yAxis = d3
+      .axisLeft()
+      .ticks(7)
+      .scale(yScale);
 
     // adding axes is also simpler now, just translate x-axis to (0,height)
     // and it's alread defined to be a bottom axis.
@@ -308,6 +338,41 @@ class Scatterplot extends React.Component {
       .text(scatterplotXXaccessor);
   }
 
+  renderPoints(
+    regl,
+    drawPoints,
+    flagBuffer,
+    colorBuffer,
+    pointBuffer,
+    projectionTF
+  ) {
+    if (!this.reglCanvas) return;
+    const { universe, responsive } = this.props;
+    // The viewport dimension is used to scale points, so we want to pass
+    // the dimension of the MAIN viewport, not the scatterplot viewport.
+    // Slightly hacky, but we want all points to scale uniformly.  Perhaps
+    // this should move to the redux state and be shared?
+    const { width: cvWidth, height: cvHeight } = responsive;
+    regl.poll();
+    regl.clear({
+      depth: 1,
+      color: [1, 1, 1, 1]
+    });
+    drawPoints({
+      flag: flagBuffer,
+      color: colorBuffer,
+      position: pointBuffer,
+      projection: projectionTF,
+      count: this.count,
+      nPoints: universe.nObs,
+      minViewportDimension: Math.min(
+        cvWidth - globals.leftSidebarWidth || width,
+        cvHeight || height
+      )
+    });
+    regl._gl.flush();
+  }
+
   render() {
     const { dispatch } = this.props;
     const { minimized } = this.state;
@@ -345,14 +410,12 @@ class Scatterplot extends React.Component {
           <Button
             type="button"
             minimal
-            onClick={() => {
+            data-testid="clear-scatterplot"
+            onClick={() =>
               dispatch({
                 type: "clear scatterplot"
-              });
-              dispatch({
-                type: "reset colorscale"
-              });
-            }}
+              })
+            }
           >
             remove
           </Button>
@@ -368,8 +431,9 @@ class Scatterplot extends React.Component {
           <canvas
             width={width}
             height={height}
+            data-testid="scatterplot"
             style={{
-              marginLeft: margin.left - 7,
+              marginLeft: margin.left,
               marginTop: margin.top
             }}
             ref={canvas => {

client/src/components/scatterplot/setupScatterplot.js

@@ -14,6 +14,7 @@ const setupScatterplot = (width, height, margin) => {
     .append("svg")
     .attr("width", width + margin.left + margin.right)
     .attr("height", height + margin.top + margin.bottom)
+    .attr("data-testid", "scatterplot-svg")
     .append("g")
     .attr("transform", `translate(${margin.left},${margin.top})`);
 

client/src/globals.js

@@ -1,38 +1,11 @@
 import { Colors } from "@blueprintjs/core";
 
-// jshint esversion: 6
-/* these will be either (preferably) specified or inferred */
-export const categories = [
-  "Sample.type",
-  "Selection",
-  "Location",
-  "Sample.name",
-  "Class",
-  "Neoplastic"
-];
-export const continuous = [
-  "Total_reads",
-  "Unique_reads",
-  "Unique_reads_percent",
-  "ERCC_reads",
-  "Non_ERCC_reads",
-  "ERCC_to_non_ERCC",
-  "Genes_detected",
-  "Multimapping_reads_percent",
-  "Splice_sites_AT.AC",
-  "Splice_sites_Annotated",
-  "Splice_sites_GC.AG",
-  "Splice_sites_GT.AG",
-  "Splice_sites_non_canonical",
-  "Splice_sites_total",
-  "Unmapped_mismatch",
-  "Unmapped_other",
-  "Unmapped_short"
-];
-
 /* if a categorical metadata field has more options than this, truncate */
 export const maxCategoricalOptionsToDisplay = 100;
 
+/* default "unassigned" value for user-created categorical metadata */
+export const unassignedCategoryLabel = "unassigned";
+
 /*
 these are default values for configuration  the CLI may supply.
 See the REST API and CLI specs for more info.
@@ -41,8 +14,11 @@ export const configDefaults = {
   features: {},
   displayNames: {},
   parameters: {
-    "max-category-items": 1000
-  }
+    "max-category-items": 1000,
+    "disable-diffexp": false,
+    "diffexp-may-be-slow": false
+  },
+  links: {}
 };
 
 /* colors */
@@ -61,6 +37,7 @@ export const darkGreen = "#448C4D";
 
 export const nonFiniteCellColor = lightGrey;
 export const defaultCellColor = "rgb(0,0,0,1)";
+export const logoColor = "black"; /* logo pink: "#E9429A" */
 
 /* typography constants */
 
@@ -76,6 +53,8 @@ export const maxParagraphWidth = 600;
 export const cellxgeneTitleLeftPadding = 14;
 export const cellxgeneTitleTopPadding = 7;
 
+export const datasetTitleMaxCharacterCount = 25;
+
 export const maxControlsWidth = 800;
 
 export const graphMargin = { top: 20, right: 10, bottom: 30, left: 40 };
@@ -91,6 +70,12 @@ export const leftSidebarSectionHeading = {
   letterSpacing: ".05em"
 };
 export const leftSidebarSectionPadding = 10;
+export const categoryLabelDisplayStringLongLength = 35;
+export const categoryLabelDisplayStringShortLength = 15;
+
+/* various timing-related behaviors */
+export const tooltipHoverOpenDelay = 1000; /* ms delay before a tooltip displays */
+export const tooltipHoverOpenDelayQuick = 500;
 
 let _API = {
   // prefix: "http://api.clustering.czi.technology/api/",
@@ -102,114 +87,3 @@ let _API = {
 
 if (window.CELLXGENE && window.CELLXGENE.API) _API = window.CELLXGENE.API;
 export const API = _API;
-
-export const ordinalColors = [
-  "#0ac115",
-  "#c10ab6",
-  "#c1710a",
-  "#0a5ac1",
-  "#c1150a",
-  "#0ab6c1",
-  "#5ac10a",
-  "#710ac1",
-  "#0ac171",
-  "#c10a5a",
-  "#b6c10a",
-  "#150ac1",
-  "#b2ffb7",
-  "#ffb2fa",
-  "#ffddb2",
-  "#b2d4ff",
-  "#ffb7b2",
-  "#b2faff",
-  "#d4ffb2",
-  "#ddb2ff",
-  "#b2ffdd",
-  "#ffb2d4",
-  "#faffb2",
-  "#b7b2ff",
-  "#27a908",
-  "#8b08a9",
-  "#a93a08",
-  "#0877a9",
-  "#a90827",
-  "#08a98b",
-  "#77a908",
-  "#3a08a9",
-  "#08a93a",
-  "#a90877",
-  "#a98b08",
-  "#0827a9",
-  "#00ff0f",
-  "#ff00ef",
-  "#ff8e00",
-  "#0070ff",
-  "#ff0f00",
-  "#00efff",
-  "#70ff00",
-  "#8e00ff",
-  "#00ff8e",
-  "#ff0070",
-  "#efff00",
-  "#0f00ff",
-  "#006606",
-  "#66005f",
-  "#663900",
-  "#002c66",
-  "#660600",
-  "#005f66",
-  "#2c6600",
-  "#390066",
-  "#006639",
-  "#66002c",
-  "#5f6600",
-  "#060066",
-  "#83ff65",
-  "#e165ff",
-  "#ff9565",
-  "#65cfff",
-  "#ff6583",
-  "#65ffe1",
-  "#cfff65",
-  "#9565ff",
-  "#65ff95",
-  "#ff65cf",
-  "#ffe165",
-  "#6583ff",
-  "#009909",
-  "#99008f",
-  "#995500",
-  "#004399",
-  "#990900",
-  "#008f99",
-  "#439900",
-  "#550099",
-  "#009955",
-  "#990043",
-  "#8f9900",
-  "#090099",
-  "#d9fecc",
-  "#f1ccfe",
-  "#fed7cc",
-  "#ccf3fe",
-  "#feccd9",
-  "#ccfef1",
-  "#f3fecc",
-  "#d7ccfe",
-  "#ccfed7",
-  "#feccf3",
-  "#fef1cc",
-  "#ccd9fe",
-  "#47ea51",
-  "#ea47e0",
-  "#eaa247",
-  "#478fea",
-  "#ea5147",
-  "#47e0ea",
-  "#8fea47",
-  "#a247ea",
-  "#47eaa2",
-  "#ea478f",
-  "#e0ea47",
-  "#5147ea"
-];

client/src/middleware/updateCellColors.js

@@ -1,150 +0,0 @@
-// jshint esversion: 6
-import _ from "lodash";
-import * as d3 from "d3";
-import { interpolateRainbow, interpolateCool } from "d3-scale-chromatic";
-import * as globals from "../globals";
-import parseRGB from "../util/parseRGB";
-import finiteExtent from "../util/finiteExtent";
-
-/*
-  https://medium.com/@jacobp100/you-arent-using-redux-middleware-enough-94ffe991e6
-  storeInstance =>
-    functionToCallWithAnActionThatWillSendItToTheNextMiddleware =>
-    actionThatDispatchWasCalledWith =>
-    valueToUseAsTheReturnValueOfTheDispatchCall
-*/
-
-/*
-  What this file does:
-
-  1. fire a filter action anywhere in the app
-  2. ** this middleware checks to see the state of all the currently selected filters,
-     including the new one
-  3. ** create updated selection from a copy of all the cells presently on the client
-     (this may be a subset of 'all')
-  4. ** append that new selection to the action so that it magically appears in the reducer
-     just because the action was fired
-
-  This is nice because we keep a lot of filtering business logic centralized
-  (what it means in practice to be selected)
-*/
-
-const updateCellColorsMiddleware = store => next => action => {
-  const s = store.getState();
-
-  /*
-  this is a hardcoded map of the things we need to keep an eye on and update
-  global cell selection in response to
-  */
-  const filterJustChanged =
-    action.type === "color by expression" ||
-    action.type === "color by continuous metadata" ||
-    action.type === "color by categorical metadata";
-
-  const obsAnnotations = _.get(s.controls, "world.obsAnnotations", null);
-  if (!filterJustChanged || !obsAnnotations) {
-    return next(
-      action
-    ); /* if the cells haven't loaded or the action wasn't a color change, bail */
-  }
-
-  let colorScale;
-  const colorsByRGB = new Array(obsAnnotations.length);
-
-  /*
-  in plain language...
-   (a) once the cells have loaded.
-   (b) each time a user changes a color control we need to update cellsMetadata colors
-  This is available to all the draw functions as controls.colorRGB[index]
-  */
-
-  if (action.type === "color by categorical metadata") {
-    const { categories } = _.filter(s.controls.world.schema.annotations.obs, {
-      name: action.colorAccessor
-    })[0];
-
-    colorScale = d3
-      .scaleSequential(interpolateRainbow)
-      .domain([0, categories.length]);
-
-    /* pre-create colors - much faster than doing it for each obs */
-    const colors = _.transform(categories, (acc, cat, idx) => {
-      acc[cat] = parseRGB(colorScale(idx));
-    });
-
-    const key = action.colorAccessor;
-    const col = obsAnnotations.col(key).asArray();
-    for (let i = 0, len = obsAnnotations.length; i < len; i += 1) {
-      const cat = col[i];
-      colorsByRGB[i] = colors[cat];
-    }
-  }
-
-  if (action.type === "color by continuous metadata") {
-    const colorBins = 100;
-    const { min, max } = action.rangeForColorAccessor;
-    colorScale = d3
-      .scaleQuantile()
-      .domain([min, max])
-      .range(_.range(colorBins - 1, -1, -1));
-
-    /* pre-create colors - much faster than doing it for each obs */
-    const colors = new Array(colorBins);
-    for (let i = 0; i < colorBins; i += 1) {
-      colors[i] = parseRGB(interpolateCool(i / colorBins));
-    }
-
-    const key = action.colorAccessor;
-    const nonFiniteColor = parseRGB(globals.nonFiniteCellColor);
-    const col = obsAnnotations.col(key).asArray();
-    for (let i = 0, len = obsAnnotations.length; i < len; i += 1) {
-      const val = col[i];
-      if (Number.isFinite(val)) {
-        const c = colorScale(val);
-        colorsByRGB[i] = colors[c];
-      } else {
-        colorsByRGB[i] = nonFiniteColor;
-      }
-    }
-  }
-
-  if (action.type === "color by expression") {
-    const { gene, data } = action;
-    const expression = data[gene]; // Float32Array
-    const colorBins = 100;
-    const [min, max] = finiteExtent(expression);
-    colorScale = d3
-      .scaleQuantile()
-      .domain([min, max])
-      .range(_.range(colorBins - 1, -1, -1));
-
-    /* pre-create colors - much faster than doing it for each obs */
-    const colors = new Array(colorBins);
-    for (let i = 0; i < colorBins; i += 1) {
-      colors[i] = parseRGB(interpolateCool(i / colorBins));
-    }
-    const nonFiniteColor = parseRGB(globals.nonFiniteCellColor);
-
-    for (let i = 0, len = expression.length; i < len; i += 1) {
-      const e = expression[i];
-      if (Number.isFinite(e)) {
-        const c = colorScale(e);
-        colorsByRGB[i] = colors[c];
-      } else {
-        colorsByRGB[i] = nonFiniteColor;
-      }
-    }
-  }
-
-  /*
-  append the result of all the filters to the action the user just triggered
-  */
-  const modifiedAction = Object.assign({}, action, {
-    colors: { rgb: colorsByRGB },
-    colorScale
-  });
-
-  return next(modifiedAction);
-};
-
-export default updateCellColorsMiddleware;

client/src/middleware/updateURLMiddleware.js

@@ -1,87 +0,0 @@
-// jshint esversion: 6
-// import uri from "urijs";
-
-/*
-NOTE: file currently not used, but retained as we expect to reinstate features in this
-area shortly.
-*/
-
-/*
-  https://medium.com/@jacobp100/you-arent-using-redux-middleware-enough-94ffe991e6
-  storeInstance
-    => functionToCallWithAnActionThatWillSendItToTheNextMiddleware
-    => actionThatDispatchWasCalledWith
-    => valueToUseAsTheReturnValueOfTheDispatchCall
-*/
-
-const updateURLMiddleware = (/* store */) => next => action => {
-  // const oldState = store.getState();
-  const nextAction = next(action);
-
-  if (action.type === "url changed") {
-    /* we don't handle pop state here - we handle it in the url reducer */
-    return nextAction;
-  }
-
-  // const state = store.getState();
-
-  /************************************************************************
-      *************************************************************************
-      1. Redux app state just changed. Clear URL, and then update it.
-      1a. We get the whole state tree to construct the url!
-      1b. But (see reducers/url.js) we try to centralize it because...
-      1c. ...the back button / initial load case ('url changed' return above)
-      means that we have to listen for 'url changed' and construct state
-      from the browser
-      *************************************************************************
-      ************************************************************************/
-
-  // const oldURI = URI(window.location.href)
-  // const newURI = URI(oldURI).setQuery({})
-
-  // if (window.location.search === "") {
-  //   newURL = uri.addQuery(category, value).toString(); /* #1 */
-  // } else if (uri.hasQuery(category, value) || uri.hasQuery(category, value, true)) { /* true param here means check arrays as well http://medialize.github.io/URI.js/docs.html#search-has */
-  //   newURL = uri.removeQuery(category, value).toString(); /* #4 */
-  // } else {
-  //   newURL = uri.addQuery(category, value).toString(); /* #2 & #3 are handled by URI */
-  // }
-  //
-  // window.history.pushState("", "", newURL)
-
-  //
-  // // Internal helper for working with URIs
-  // const oldURI = new URI(window.location.href);
-  // const newURI = new URI(oldURI).setQueryData({});
-  //
-  // newURI.setPath('/foo/bar');
-  //
-  // // Set the path based on state
-  // if (!state.isOnLandingPage && state.project.id) {
-  //   newURI.setPath(newURI.getPath() + state.project.id + '/');
-  //   newURI.addQueryData('baz', state.mode);
-  //   newURI.addQueryData('bat', state.selection.activePageID);
-  // } else {
-  //   newURI.setPath(newURI.getPath() + state.landingSection + '/');
-  // }
-  //
-  // // Avoid URL thrashing by replacing state while loading instead of pushing
-  // const newPath = newURI.toString();
-  // const oldPath = oldURI.toString();
-  // if (newPath !== oldPath) {
-  //   if (
-  //     (oldState.mode === 'asdf' &&
-  //       state.mode === 'asdf' &&
-  //       !oldState.isOnLandingPage) ||
-  //     oldState.isLoadingProject !== state.isLoadingProject
-  //   ) {
-  //     window.history.replaceState(null, null, newPath);
-  //   } else {
-  //     window.history.pushState(null, null, newPath);
-  //   }
-  // }
-
-  return nextAction;
-};
-
-export default updateURLMiddleware;

client/src/reducers/annotations.js

@@ -0,0 +1,79 @@
+/*
+Reducers for annotation UI-state.
+*/
+const Annotations = (
+  state = {
+    isEditingCategoryName: false,
+    isEditingLabelName: false,
+    categoryBeingEdited: false,
+    labelEditable: { category: null, label: null }
+  },
+  action
+) => {
+  switch (action.type) {
+    /* CATEGORY */
+    case "annotation: activate add new label mode":
+      return {
+        ...state,
+        isAddingNewLabel: true,
+        categoryAddingNewLabel: action.data
+      };
+    case "annotation: disable add new label mode":
+      return {
+        ...state,
+        isAddingNewLabel: false,
+        categoryAddingNewLabel: null
+      };
+    case "annotation: add new label to category":
+      return {
+        ...state,
+        isAddingNewLabel: false,
+        categoryAddingNewLabel: null
+      };
+    case "annotation: activate category edit mode":
+      return {
+        ...state,
+        isEditingCategoryName: true,
+        categoryBeingEdited: action.data
+      };
+    case "annotation: disable category edit mode":
+      return {
+        ...state,
+        isEditingCategoryName: false,
+        categoryBeingEdited: null
+      };
+    case "annotation: category edited":
+      return {
+        ...state,
+        isEditingCategoryName: true,
+        categoryBeingEdited: null
+      };
+
+    /* LABEL */
+    case "annotation: activate edit label mode":
+      return {
+        ...state,
+        isEditingLabelName: true,
+        labelEditable: {
+          category: action.metadataField,
+          label: action.categoryIndex
+        }
+      };
+    case "annotation: cancel edit label mode":
+      return {
+        ...state,
+        isEditingLabelName: false,
+        labelEditable: { category: null, label: null }
+      };
+    case "annotation: label edited":
+      return {
+        ...state,
+        isEditingLabelName: false,
+        labelEditable: { category: null, label: null }
+      };
+    default:
+      return state;
+  }
+};
+
+export default Annotations;

client/src/reducers/autosave.js

@@ -0,0 +1,53 @@
+const Autosave = (
+	state = {
+		saveInProgress: false,
+		error: false,
+		lastSavedObsAnnotations: null
+	},
+	action,
+	nextSharedState
+) => {
+	switch (action.type) {
+		case "initial data load complete (universe exists)": {
+			/* don't save on init */
+			const { universe } = nextSharedState;
+			return {
+				...state,
+				error: false,
+				saveInProgress: false,
+				lastSavedObsAnnotations: universe.obsAnnotations
+			};
+		}
+
+		case "writable obs annotations - save started": {
+			return {
+				...state,
+				saveInProgress: true
+			};
+		}
+
+		case "writable obs annotations - save error": {
+			const { message } = action;
+			return {
+				...state,
+				error: message,
+				saveInProgress: false
+			};
+		}
+
+		case "writable obs annotations - save complete": {
+			const lastSavedObsAnnotations = action.obsAnnotations;
+			return {
+				...state,
+				saveInProgress: false,
+				error: false,
+				lastSavedObsAnnotations
+			};
+		}
+
+		default:
+			return { ...state };
+	}
+};
+
+export default Autosave;

client/src/reducers/cascade.js

@@ -0,0 +1,44 @@
+export default function cascadeReducers(arg) {
+  /*
+  Combined a set of cascading reducers into a single reducer.  Cascading
+  reducers are reducers which may rely on state computed by another reducer.
+  Therefore, they:
+  - must be composed in a particular order (currently, this is a simple
+    linear list of reducers, run in list order)
+  - must have access to partially updated "next state" so they can further
+    derive state.
+
+  Parameter is one of:
+  - a Map object
+  - an array of tuples, [ [key1, reducer1], [key2, reducer2], ... ]
+  Ie,  cascadeReducers([ ["a", reduceA], ["b", reduceB] ])
+
+  Each reducer will be called with the sigature:
+      (prevState, action, sharedNextState, sharedPrevState) => newState
+
+  cascadeReducers will build a composite newState object, much
+  like combinedReducers.  Additional semantics:
+  - reducers guaranteed to be called in order
+  - each reducer will receive shared objects
+  */
+  const reducers = arg instanceof Map ? arg : new Map(arg);
+  const reducerKeys = [...reducers.keys()];
+  return (prevState, action) => {
+    const nextState = {};
+    let stateChange = false;
+    for (let i = 0, l = reducerKeys.length; i < l; i += 1) {
+      const key = reducerKeys[i];
+      const reducer = reducers.get(key);
+      const prevStateForKey = prevState ? prevState[key] : undefined;
+      const nextStateForKey = reducer(
+        prevStateForKey,
+        action,
+        nextState,
+        prevState
+      );
+      nextState[key] = nextStateForKey;
+      stateChange = stateChange || nextStateForKey !== prevStateForKey;
+    }
+    return stateChange ? nextState : prevState;
+  };
+}

client/src/reducers/categoricalSelection.js

@@ -0,0 +1,143 @@
+import { ControlsHelpers as CH } from "../util/stateManager";
+import * as globals from "../globals";
+
+function maxCategoryItems(state) {
+  return (
+    state.config.parameters?.["max-category-items"] ??
+    globals.configDefaults.parameters["max-category-items"]
+  );
+}
+
+const CategoricalSelection = (
+  state,
+  action,
+  nextSharedState,
+  prevSharedState
+) => {
+  switch (action.type) {
+    case "initial data load complete (universe exists)":
+    case "set World to current selection":
+    case "reset World to eq Universe":
+    case "set clip quantiles": {
+      const { world } = nextSharedState;
+      const newState = CH.createCategoricalSelection(
+        world,
+        CH.selectableCategoryNames(world, maxCategoryItems(prevSharedState))
+      );
+      return newState;
+    }
+
+    case "categorical metadata filter select": {
+      /*
+      Set the specific category in this field to false
+      */
+      const newCategoryValueSelected = Array.from(
+        state[action.metadataField].categoryValueSelected
+      );
+      newCategoryValueSelected[action.categoryIndex] = true;
+      const newCategoricalSelection = {
+        ...state,
+        [action.metadataField]: {
+          ...state[action.metadataField],
+          categoryValueSelected: newCategoryValueSelected
+        }
+      };
+      return newCategoricalSelection;
+    }
+
+    case "categorical metadata filter deselect": {
+      /*
+      Set the specific category in this field to false
+      */
+      const newCategoryValueSelected = Array.from(
+        state[action.metadataField].categoryValueSelected
+      );
+      newCategoryValueSelected[action.categoryIndex] = false;
+      const newCategoricalSelection = {
+        ...state,
+        [action.metadataField]: {
+          ...state[action.metadataField],
+          categoryValueSelected: newCategoryValueSelected
+        }
+      };
+      return newCategoricalSelection;
+    }
+
+    case "categorical metadata filter none of these": {
+      /*
+      set all categories in this field to false.
+      */
+      const newCategoricalSelection = {
+        ...state,
+        [action.metadataField]: {
+          ...state[action.metadataField],
+          categorySelected: false,
+          categoryValueSelected: Array.from(
+            state[action.metadataField].categoryValueSelected
+          ).fill(false)
+        }
+      };
+      return newCategoricalSelection;
+    }
+
+    case "categorical metadata filter all of these": {
+      /*
+      set all categories in this field to true.
+      */
+      const newCategoricalSelection = {
+        ...state,
+        [action.metadataField]: {
+          ...state[action.metadataField],
+          categorySelected: true,
+          categoryValueSelected: Array.from(
+            state[action.metadataField].categoryValueSelected
+          ).fill(true)
+        }
+      };
+      return newCategoricalSelection;
+    }
+
+    case "annotation: create category": {
+      const { world } = nextSharedState;
+      const name = action.data;
+      return {
+        ...state,
+        ...CH.createCategoricalSelection(world, [name])
+      };
+    }
+
+    case "annotation: category edited": {
+      const name = action.metadataField;
+      const newName = action.newCategoryText;
+      const { [name]: catSeln, ...newState } = state;
+      newState[newName] = catSeln;
+      return newState;
+    }
+
+    case "annotation: delete category": {
+      const name = action.metadataField;
+      const { [name]: _, ...newState } = state;
+      return newState;
+    }
+
+    case "annotation: label current cell selection":
+    case "annotation: add new label to category":
+    case "annotation: label edited":
+    case "annotation: delete label": {
+      /* need to rebuild the state for this annotation */
+      const { world } = nextSharedState;
+      const name = action.metadataField;
+      const { [name]: _, ...partialState } = state;
+      return {
+        ...partialState,
+        ...CH.createCategoricalSelection(world, [name])
+      };
+    }
+
+    default: {
+      return state;
+    }
+  }
+};
+
+export default CategoricalSelection;

client/src/reducers/centroidLabel.js

@@ -0,0 +1,44 @@
+import calcCentroid from "../util/centroid";
+
+const initialState = {
+  metadataField: "",
+  categoryIndex: -1,
+  categoryField: "",
+  centroidXY: [-1, -1]
+};
+
+const CentroidLabel = (state = initialState, action, sharedNextState) => {
+  const { categoricalSelection, world, layoutChoice } = sharedNextState;
+  const { metadataField, categoryIndex } = action;
+  const categoryField =
+    categoricalSelection?.[metadataField]?.categoryValues[categoryIndex];
+  switch (action.type) {
+    case "category value mouse hover start":
+      return {
+        ...state,
+        metadataField,
+        categoryIndex,
+        categoryField,
+        centroidXY: null /* calcCentroid(  This function call is computationally heavy and also leading to large GC. Before reimplementation, look into optimization and memoization
+          world,
+          metadataField,
+          categoryField,
+          layoutChoice.currentDimNames
+        ) */
+      };
+
+    case "category value mouse hover end":
+      if (
+        metadataField === state.metadataField &&
+        categoryIndex === state.categoryIndex
+      ) {
+        return initialState;
+      }
+      return state;
+
+    default:
+      return state;
+  }
+};
+
+export default CentroidLabel;

client/src/reducers/colors.js

@@ -0,0 +1,151 @@
+import { ColorHelpers } from "../util/stateManager";
+
+const ColorsReducer = (
+  state = {
+    colorMode: null,
+    colorAccessor: null,
+    rgb: null,
+    scale: null
+  },
+  action,
+  nextSharedState,
+  prevSharedState
+) => {
+  switch (action.type) {
+    case "initial data load complete (universe exists)":
+    case "reset World to eq Universe": {
+      const { world } = nextSharedState;
+      const colorMode = null;
+      const colorAccessor = null;
+      const { rgb, scale } = ColorHelpers.createColors(world, colorMode);
+      return {
+        ...state,
+        colorAccessor,
+        colorMode,
+        rgb,
+        scale
+      };
+    }
+
+    case "set clip quantiles":
+    case "set World to current selection": {
+      const { world: prevWorld, controls: prevControls } = prevSharedState;
+      const resetColorState = ColorHelpers.checkIfColorByDiffexpAndResetColors(
+        prevControls,
+        state,
+        prevWorld
+      );
+      if (resetColorState) {
+        return resetColorState;
+      }
+
+      const { colorMode, colorAccessor } = state;
+      const { world } = nextSharedState;
+      const { rgb, scale } = ColorHelpers.createColors(
+        world,
+        colorMode,
+        colorAccessor
+      );
+      return {
+        ...state,
+        rgb,
+        scale
+      };
+    }
+
+    case "reset colorscale": {
+      return {
+        ...state,
+        ...ColorHelpers.resetColors(prevSharedState.world)
+      };
+    }
+
+    case "color by categorical metadata":
+    case "color by continuous metadata": {
+      const { world } = prevSharedState;
+
+      /* toggle between this mode and reset */
+      const resetCurrent =
+        action.type === state.colorMode &&
+        action.colorAccessor === state.colorAccessor;
+      const colorMode = !resetCurrent ? action.type : null;
+      const colorAccessor = !resetCurrent ? action.colorAccessor : null;
+
+      const { rgb, scale } = ColorHelpers.createColors(
+        world,
+        colorMode,
+        colorAccessor
+      );
+      return {
+        ...state,
+        colorMode,
+        colorAccessor,
+        rgb,
+        scale
+      };
+    }
+
+    case "color by expression": {
+      const { world } = prevSharedState;
+
+      /* toggle between this mode and reset */
+      const resetCurrent =
+        action.type === state.colorMode && action.gene === state.colorAccessor;
+      const colorMode = !resetCurrent ? action.type : null;
+      const colorAccessor = !resetCurrent ? action.gene : null;
+
+      const { rgb, scale } = ColorHelpers.createColors(
+        world,
+        colorMode,
+        colorAccessor
+      );
+      return {
+        ...state,
+        colorMode,
+        colorAccessor,
+        rgb,
+        scale
+      };
+    }
+
+    case "annotation: add new label to category":
+    case "annotation: label current cell selection":
+    case "annotation: delete label": {
+      const { world } = nextSharedState;
+      const { colorMode, colorAccessor } = state;
+      const { metadataField } = action;
+      if (
+        colorMode !== "color by categorical metadata" ||
+        colorAccessor !== metadataField
+      )
+        return state;
+
+      /* else, we need to rebuild colors as labels have changed! */
+      const { rgb, scale } = ColorHelpers.createColors(
+        world,
+        colorMode,
+        colorAccessor
+      );
+      return { ...state, rgb, scale };
+    }
+
+    case "clear differential expression": {
+      const { world: prevWorld, controls: prevControls } = prevSharedState;
+      const resetColorState = ColorHelpers.checkIfColorByDiffexpAndResetColors(
+        prevControls,
+        state,
+        prevWorld
+      );
+      if (resetColorState) {
+        return resetColorState;
+      }
+      return state;
+    }
+
+    default: {
+      return state;
+    }
+  }
+};
+
+export default ColorsReducer;

client/src/reducers/continuousSelection.js

@@ -0,0 +1,35 @@
+import { makeContinuousDimensionName } from "../util/nameCreators";
+
+const ContinuousSelection = (state = {}, action) => {
+  switch (action.type) {
+    case "reset World to eq Universe":
+    case "set clip quantiles": {
+      return {};
+    }
+    case "continuous metadata histogram start":
+    case "continuous metadata histogram brush":
+    case "continuous metadata histogram end": {
+      const name = makeContinuousDimensionName(
+        action.continuousNamespace,
+        action.selection
+      );
+      return {
+        ...state,
+        [name]: action.range
+      };
+    }
+    case "continuous metadata histogram cancel": {
+      const name = makeContinuousDimensionName(
+        action.continuousNamespace,
+        action.selection
+      );
+      const { [name]: deletedField, ...newState } = state;
+      return newState;
+    }
+    default: {
+      return state;
+    }
+  }
+};
+
+export default ContinuousSelection;

client/src/reducers/controls.js

@@ -2,56 +2,29 @@
 
 import _ from "lodash";
 
-import { World, WorldUtil, ControlsHelper } from "../util/stateManager";
-import parseRGB from "../util/parseRGB";
-import Crossfilter from "../util/typedCrossfilter";
-import * as globals from "../globals";
-import {
-  layoutDimensionName,
-  obsAnnoDimensionName,
-  userDefinedDimensionName,
-  diffexpDimensionName,
-  makeContinuousDimensionName
-} from "../util/nameCreators";
-
 const Controls = (
   state = {
     // data loading flag
-    loading: false,
+    loading: true,
     error: null,
 
-    // configuration
-    maxCategoryItems: globals.configDefaults.parameters["max-category-items"],
-
-    // the whole big bang
-    universe: null,
-    fullUniverseCache: null,
-
     // all of the data + selection state
-    world: null,
-    colorRGB: null,
-    categoricalSelectionState: null,
-    crossfilter: null,
-    dimensionMap: null,
     userDefinedGenes: [],
     userDefinedGenesLoading: false,
     diffexpGenes: [],
 
-    colorAccessor: null,
-    colorScale: null,
     resettingInterface: false,
-
+    graphInteractionMode: "select",
     opacityForDeselectedCells: 0.2,
-    graphBrushSelection: null,
-    continuousSelection: null,
     scatterplotXXaccessor: null, // just easier to read
     scatterplotYYaccessor: null,
-    axesHaveBeenDrawn: false,
     graphRenderCounter: 0 /* integer as <Component key={graphRenderCounter} - a change in key forces a remount */,
     __storedStateForCelllist1__: null /* will need procedural control of brush ie., brush.extent https://bl.ocks.org/micahstubbs/3cda05ca68cba260cb81 */,
     __storedStateForCelllist2__: null
   },
-  action
+  action,
+  nextSharedState,
+  prevSharedState
 ) => {
   /*
   For now, log anything looking like an error to the console.
@@ -65,199 +38,29 @@ const Controls = (
           Initialization, World/Universe management
           and data loading.
     ******************************************************/
-    case "configuration load complete": {
-      // there are a couple of configuration items we need to retain
-      return {
-        ...state,
-        maxCategoryItems: _.get(
-          state.config,
-          "parameters.max-category-items",
-          globals.configDefaults.parameters["max-category-items"]
-        )
-      };
-    }
     case "initial data load start": {
       return { ...state, loading: true };
     }
     case "initial data load complete (universe exists)": {
       /* first light - create world & other data-driven defaults */
-      const { universe } = action;
-      const world = World.createWorldFromEntireUniverse(universe);
-      const colorRGB = new Array(universe.nObs).fill(
-        parseRGB(globals.defaultCellColor)
-      );
-      const categoricalSelectionState = ControlsHelper.createCategoricalSelectionState(
-        state,
-        world
-      );
-      const crossfilter = Crossfilter(world.obsAnnotations);
-      const dimensionMap = World.createObsDimensionMap(crossfilter, world);
-      WorldUtil.clearCaches();
-
       return {
         ...state,
         loading: false,
         error: null,
-        universe,
-        fullUniverseCache: { world, crossfilter, dimensionMap },
-        world,
-        colorRGB,
-        categoricalSelectionState,
-        crossfilter,
-        dimensionMap,
-        colorAccessor: null,
         resettingInterface: false
       };
     }
     case "reset World to eq Universe": {
-      const {
-        userDefinedGenes,
-        diffexpGenes,
-        universe,
-        fullUniverseCache
-      } = state;
-      const { world, crossfilter } = fullUniverseCache;
-      // reset all crossfilter dimensions
-      _.forEach(fullUniverseCache.dimensionMap, dim => dim.filterAll());
-      const colorRGB = new Array(universe.nObs).fill(
-        parseRGB(globals.defaultCellColor)
-      );
-      const categoricalSelectionState = ControlsHelper.createCategoricalSelectionState(
-        state,
-        world
-      );
-
-      /* free dimensions not in cache (otherwise they leak) */
-      _.forEach(state.dimensionMap, (dim, dimName) => {
-        if (!fullUniverseCache.dimensionMap[dimName]) {
-          dim.dispose();
-        }
-      });
-      const dimensionMap = {
-        ...fullUniverseCache.dimensionMap,
-        ...ControlsHelper.createGenesDimMap(
-          userDefinedGenes,
-          diffexpGenes,
-          world,
-          crossfilter
-        )
-      };
-      WorldUtil.clearCaches();
-
       return {
         ...state,
-        world,
-        colorRGB,
-        categoricalSelectionState,
-        crossfilter,
-        dimensionMap,
-        colorAccessor: null,
         resettingInterface: false
       };
     }
     case "set World to current selection": {
-      const { userDefinedGenes, diffexpGenes } = state;
-
-      /* Set viewable world to be the currently selected data */
-      const world = World.createWorldFromCurrentSelection(
-        action.universe,
-        action.world,
-        action.crossfilter
-      );
-      const colorRGB = new Array(world.nObs).fill(
-        parseRGB(globals.defaultCellColor)
-      );
-      const categoricalSelectionState = ControlsHelper.createCategoricalSelectionState(
-        state,
-        world
-      );
-      const crossfilter = Crossfilter(world.obsAnnotations);
-      const dimensionMap = {
-        ...World.createObsDimensionMap(crossfilter, world),
-        ...ControlsHelper.createGenesDimMap(
-          userDefinedGenes,
-          diffexpGenes,
-          world,
-          crossfilter
-        )
-      };
-      WorldUtil.clearCaches();
-
       return {
         ...state,
         loading: false,
-        error: null,
-        world,
-        colorRGB,
-        categoricalSelectionState,
-        crossfilter,
-        dimensionMap,
-        colorAccessor: null
-      };
-    }
-    case "expression load success": {
-      const { world, universe } = state;
-      let universeVarData = universe.varData;
-      let worldVarData = world.varData;
-
-      // Load new expression data into the varData dataframes, if
-      // not already present.
-      _.forEach(action.expressionData, (val, key) => {
-        // If not already in universe.varData, save entire expression column
-        if (!universeVarData.hasCol(key)) {
-          universeVarData = universeVarData.withCol(key, val);
-        }
-
-        // If not already in world.varData, save sliced expression column
-        if (!worldVarData.hasCol(key)) {
-          // Slice if world !== universe, else just use whole column.
-          // Use the obsAnnotation index as the cut key, as we keep
-          // all world dataframes in sync.
-          let worldValSlice = val;
-          if (!World.worldEqUniverse(world, universe)) {
-            worldValSlice = universeVarData
-              .subset(world.obsAnnotations.rowIndex.keys(), [key], null)
-              .icol(0)
-              .asArray();
-          }
-
-          // Now build world's varData dataframe
-          worldVarData = worldVarData.withCol(
-            key,
-            worldValSlice,
-            world.obsAnnotations.rowIndex
-          );
-        }
-      });
-
-      // Prune size of varData "cache" if getting out of hand....
-      const { userDefinedGenes, diffexpGenes } = state;
-      const allTheGenesWeNeed = _.uniq(
-        [].concat(
-          userDefinedGenes,
-          diffexpGenes,
-          Object.keys(action.expressionData)
-        )
-      );
-      universeVarData = ControlsHelper.pruneVarDataCache(
-        universeVarData,
-        allTheGenesWeNeed
-      );
-      worldVarData = ControlsHelper.pruneVarDataCache(
-        worldVarData,
-        allTheGenesWeNeed
-      );
-
-      return {
-        ...state,
-        universe: {
-          ...universe,
-          varData: universeVarData
-        },
-        world: {
-          ...world,
-          varData: worldVarData
-        }
+        error: null
       };
     }
     case "request user defined gene started": {
@@ -273,114 +76,51 @@ const Controls = (
       };
     }
     case "request user defined gene success": {
-      const { world, crossfilter, dimensionMap, userDefinedGenes } = state;
-      const _userDefinedGenes = userDefinedGenes.slice();
-      const gene = action.data.genes[0];
-
-      dimensionMap[
-        userDefinedDimensionName(gene)
-      ] = World.createVarDataDimension(world, crossfilter, gene);
-
+      const { userDefinedGenes } = state;
+      const _userDefinedGenes = _.uniq(
+        userDefinedGenes.concat(action.data.genes)
+      );
       return {
         ...state,
-        dimensionMap,
         userDefinedGenes: _userDefinedGenes,
         userDefinedGenesLoading: false
       };
     }
     case "request differential expression success": {
-      const { world, crossfilter, dimensionMap } = state;
+      const { world } = prevSharedState;
+      const varIndexName = world.schema.annotations.var.index;
       const _diffexpGenes = [];
-
       action.data.forEach(d => {
-        _diffexpGenes.push(world.varAnnotations.at(d[0], "name"));
+        _diffexpGenes.push(world.varAnnotations.at(d[0], varIndexName));
       });
-
-      _.forEach(_diffexpGenes, gene => {
-        dimensionMap[diffexpDimensionName(gene)] = World.createVarDataDimension(
-          world,
-          crossfilter,
-          gene
-        );
-      });
-
       return {
         ...state,
-        dimensionMap,
         diffexpGenes: _diffexpGenes
       };
     }
     case "clear differential expression": {
-      const { world, dimensionMap } = state;
-      const _dimensionMap = dimensionMap;
-      _.forEach(action.diffExp, values => {
-        const name = world.varAnnotations.at(values[0], "name");
-        // clean up crossfilter dimensions
-        const dimension = dimensionMap[diffexpDimensionName(name)];
-        dimension.dispose();
-        delete dimensionMap[diffexpDimensionName(name)];
-      });
       return {
         ...state,
-        dimensionMap: _dimensionMap,
         diffexpGenes: []
       };
     }
-    case "user defined gene": {
-      /*
-        this could also live in expression success with a conditional,
-        but that handles diffexp also
-      */
-      const newUserDefinedGenes = state.userDefinedGenes.slice();
-      newUserDefinedGenes.push(action.data);
-      return {
-        ...state,
-        userDefinedGenes: newUserDefinedGenes
-      };
-    }
     case "clear user defined gene": {
-      const { userDefinedGenes, dimensionMap } = state;
+      const { userDefinedGenes } = state;
       const newUserDefinedGenes = _.filter(
         userDefinedGenes,
         d => d !== action.data
       );
-
-      const dimension = dimensionMap[userDefinedDimensionName(action.data)];
-      dimension.dispose();
-      delete dimensionMap[userDefinedDimensionName(action.data)];
-
       return {
         ...state,
-        dimensionMap,
         userDefinedGenes: newUserDefinedGenes
       };
     }
     case "clear all user defined genes": {
-      const { userDefinedGenes, dimensionMap } = state;
-
-      _.forEach(userDefinedGenes, gene => {
-        const dimension = dimensionMap[userDefinedDimensionName(gene)];
-        dimension.dispose();
-        delete dimensionMap[userDefinedDimensionName(gene)];
-      });
-
       return {
         ...state,
-        dimensionMap,
         userDefinedGenes: []
       };
     }
-    case "reset colorscale": {
-      const { world } = state;
-      const colorRGB = new Array(world.nObs).fill(
-        parseRGB(globals.defaultCellColor)
-      );
-      return {
-        ...state,
-        colorRGB,
-        colorAccessor: null
-      };
-    }
     case "expression load error":
     case "initial data load error": {
       return {
@@ -393,52 +133,11 @@ const Controls = (
     /*******************************
              User Events
      *******************************/
-    case "graph brush selection change": {
-      state.dimensionMap[layoutDimensionName("XY")].filterWithinRect(
-        action.brushCoords.northwest,
-        action.brushCoords.southeast
-      );
+    case "change graph interaction mode":
       return {
         ...state,
-        graphBrushSelection: action.brushCoords
+        graphInteractionMode: action.data
       };
-    }
-    case "lasso deselect":
-    case "graph brush deselect": {
-      state.dimensionMap[layoutDimensionName("XY")].filterAll();
-      return {
-        ...state,
-        graphBrushSelection: null
-      };
-    }
-    case "lasso selection": {
-      const { polygon } = action;
-      const dXY = state.dimensionMap[layoutDimensionName("XY")];
-      if (polygon.length < 3) {
-        // single point or a line is not a polygon, and is therefore a deselect
-        dXY.filterAll();
-      } else {
-        dXY.filterWithinPolygon(polygon);
-      }
-      return {
-        ...state
-      };
-    }
-    case "continuous metadata histogram brush": {
-      const name = makeContinuousDimensionName(
-        action.continuousNamespace,
-        action.selection
-      );
-
-      // action.selection: metadata name being selected
-      // action.range: filter range, or null if deselected
-      if (!action.range) {
-        state.dimensionMap[name].filterAll();
-      } else {
-        state.dimensionMap[name].filterRange(action.range);
-      }
-      return { ...state };
-    }
     case "change opacity deselected cells in 2d graph background":
       return {
         ...state,
@@ -457,116 +156,6 @@ const Controls = (
         resettingInterface: true
       };
     }
-    /*******************************
-          Categorical metadata
-    *******************************/
-    case "categorical metadata filter select": {
-      const newCategorySelected = Array.from(
-        state.categoricalSelectionState[action.metadataField].categorySelected
-      );
-      newCategorySelected[action.categoryIndex] = true;
-      const newCategoricalSelectionState = {
-        ...state.categoricalSelectionState,
-        [action.metadataField]: {
-          ...state.categoricalSelectionState[action.metadataField],
-          categorySelected: newCategorySelected
-        }
-      };
-
-      // update the filter to match all selected options
-      const cat = newCategoricalSelectionState[action.metadataField];
-      state.dimensionMap[obsAnnoDimensionName(action.metadataField)].filterEnum(
-        ControlsHelper.selectedValuesForCategory(cat)
-      );
-
-      return {
-        ...state,
-        categoricalSelectionState: newCategoricalSelectionState
-      };
-    }
-    case "categorical metadata filter deselect": {
-      const newCategorySelected = Array.from(
-        state.categoricalSelectionState[action.metadataField].categorySelected
-      );
-      newCategorySelected[action.categoryIndex] = false;
-      const newCategoricalSelectionState = {
-        ...state.categoricalSelectionState,
-        [action.metadataField]: {
-          ...state.categoricalSelectionState[action.metadataField],
-          categorySelected: newCategorySelected
-        }
-      };
-
-      // update the filter to match all selected options
-      const cat = newCategoricalSelectionState[action.metadataField];
-      state.dimensionMap[obsAnnoDimensionName(action.metadataField)].filterEnum(
-        ControlsHelper.selectedValuesForCategory(cat)
-      );
-
-      return {
-        ...state,
-        categoricalSelectionState: newCategoricalSelectionState
-      };
-    }
-    case "categorical metadata filter none of these": {
-      const newCategoricalSelectionState = {
-        ...state.categoricalSelectionState,
-        [action.metadataField]: {
-          ...state.categoricalSelectionState[action.metadataField],
-          categorySelected: Array.from(
-            state.categoricalSelectionState[action.metadataField]
-              .categorySelected
-          ).fill(false)
-        }
-      };
-      state.dimensionMap[
-        obsAnnoDimensionName(action.metadataField)
-      ].filterNone();
-      return {
-        ...state,
-        categoricalSelectionState: newCategoricalSelectionState
-      };
-    }
-    case "categorical metadata filter all of these": {
-      const newCategoricalSelectionState = {
-        ...state.categoricalSelectionState,
-        [action.metadataField]: {
-          ...state.categoricalSelectionState[action.metadataField],
-          categorySelected: Array.from(
-            state.categoricalSelectionState[action.metadataField]
-              .categorySelected
-          ).fill(true)
-        }
-      };
-      state.dimensionMap[
-        obsAnnoDimensionName(action.metadataField)
-      ].filterAll();
-      return {
-        ...state,
-        categoricalSelectionState: newCategoricalSelectionState
-      };
-    }
-
-    /*******************************
-              Color Scale
-    *******************************/
-    case "color by categorical metadata":
-    case "color by continuous metadata": {
-      return {
-        ...state,
-        colorRGB: action.colors.rgb,
-        colorAccessor: action.colorAccessor,
-        colorScale: action.colorScale
-      };
-    }
-    case "color by expression": {
-      return {
-        ...state,
-        colorRGB: action.colors.rgb,
-        colorAccessor: action.gene,
-        colorScale: action.colorScale
-      };
-    }
 
     /*******************************
               Scatterplot

client/src/reducers/crossfilter.js

@@ -0,0 +1,277 @@
+import _ from "lodash";
+
+import Crossfilter from "../util/typedCrossfilter";
+import {
+  World,
+  ControlsHelpers as CH,
+  AnnotationsHelpers as AH
+} from "../util/stateManager";
+import {
+  layoutDimensionName,
+  obsAnnoDimensionName,
+  userDefinedDimensionName,
+  diffexpDimensionName,
+  makeContinuousDimensionName
+} from "../util/nameCreators";
+
+const XYDimName = layoutDimensionName("XY");
+
+const CrossfilterReducerBase = (
+  state = null,
+  action,
+  nextSharedState,
+  prevSharedState
+) => {
+  switch (action.type) {
+    case "initial data load complete (universe exists)": {
+      const { world, layoutChoice } = nextSharedState;
+      const crossfilter = World.createObsDimensions(
+        new Crossfilter(world.obsAnnotations),
+        world,
+        layoutChoice.currentDimNames
+      );
+      return crossfilter;
+    }
+
+    case "reset World to eq Universe": {
+      const { userDefinedGenes, diffexpGenes } = prevSharedState.controls;
+      const { world } = nextSharedState;
+      let { crossfilter } = prevSharedState.resetCache;
+      crossfilter = CH.createGeneDimensions(
+        userDefinedGenes,
+        diffexpGenes,
+        world,
+        crossfilter
+      );
+      crossfilter = AH.createWritableAnnotationDimensions(world, crossfilter);
+      return crossfilter;
+    }
+
+    case "set clip quantiles":
+    case "set World to current selection": {
+      const { userDefinedGenes, diffexpGenes } = prevSharedState.controls;
+      const { world, layoutChoice } = nextSharedState;
+      let crossfilter = new Crossfilter(world.obsAnnotations);
+      crossfilter = World.createObsDimensions(
+        crossfilter,
+        world,
+        layoutChoice.currentDimNames
+      );
+      crossfilter = CH.createGeneDimensions(
+        userDefinedGenes,
+        diffexpGenes,
+        world,
+        crossfilter
+      );
+      return crossfilter;
+    }
+
+    case "set layout choice": {
+      /*
+      when switching layouts:
+      - delete the existing XY index
+      - add the new XY index (which implicitly selects all on it)
+      */
+      const { world, layoutChoice } = nextSharedState;
+      return state
+        .delDimension(layoutDimensionName("XY"))
+        .addDimension(
+          layoutDimensionName("XY"),
+          "spatial",
+          world.obsLayout.col(layoutChoice.currentDimNames[0]).asArray(),
+          world.obsLayout.col(layoutChoice.currentDimNames[1]).asArray()
+        );
+    }
+
+    case "request user defined gene success": {
+      const { world } = prevSharedState;
+      const gene = action.data.genes[0];
+      return state.addDimension(
+        userDefinedDimensionName(gene),
+        "scalar",
+        world.varData.col(gene).asArray(),
+        Float32Array
+      );
+    }
+
+    case "request differential expression success": {
+      const { world } = prevSharedState;
+      const varIndexName = world.schema.annotations.var.index;
+      const genes = _.map(action.data, d =>
+        world.varAnnotations.at(d[0], varIndexName)
+      );
+      const crossfilter = _.reduce(
+        genes,
+        (xfltr, gene) =>
+          xfltr.addDimension(
+            diffexpDimensionName(gene),
+            "scalar",
+            world.varData.col(gene).asArray(),
+            Float32Array
+          ),
+        state
+      );
+      return crossfilter;
+    }
+
+    case "clear differential expression": {
+      const { world } = prevSharedState;
+      const varIndexName = world.schema.annotations.var.index;
+      const crossfilter = _.reduce(
+        action.diffExp,
+        (xfltr, values) => {
+          const name = world.varAnnotations.at(values[0], varIndexName);
+          return xfltr.delDimension(diffexpDimensionName(name));
+        },
+        state
+      );
+      return crossfilter;
+    }
+
+    case "clear user defined gene": {
+      return state.delDimension(userDefinedDimensionName(action.data));
+    }
+
+    case "clear all user defined genes": {
+      const { userDefinedGenes } = prevSharedState.controls;
+      const crossfilter = _.reduce(
+        userDefinedGenes,
+        (xfltr, gene) => xfltr.delDimension(userDefinedDimensionName(gene)),
+        state
+      );
+      return crossfilter;
+    }
+
+    case "annotation: create category": {
+      const name = action.data;
+      const { world } = nextSharedState;
+      const colData = world.obsAnnotations.col(name).asArray();
+      return state.addDimension(obsAnnoDimensionName(name), "enum", colData);
+    }
+
+    case "annotation: category edited": {
+      const name = action.metadataField;
+      const newName = action.newCategoryText;
+      return state.renameDimension(
+        obsAnnoDimensionName(name),
+        obsAnnoDimensionName(newName)
+      );
+    }
+
+    case "annotation: delete category": {
+      return state.delDimension(obsAnnoDimensionName(action.metadataField));
+    }
+
+    case "annotation: label current cell selection":
+    case "annotation: label edited":
+    case "annotation: delete label": {
+      /* we need to reindex the dimension.  For now, just drop it and add another */
+      const name = action.metadataField;
+      const dimName = obsAnnoDimensionName(name);
+      const { world } = nextSharedState;
+      const colData = world.obsAnnotations.col(name).asArray();
+      return state.delDimension(dimName).addDimension(dimName, "enum", colData);
+    }
+
+    case "graph brush end":
+    case "graph brush change": {
+      const [minX, maxY] = action.brushCoords.northwest;
+      const [maxX, minY] = action.brushCoords.southeast;
+      return state.select(XYDimName, {
+        mode: "within-rect",
+        minX,
+        minY,
+        maxX,
+        maxY
+      });
+    }
+
+    case "graph lasso end": {
+      const { polygon } = action;
+      return state.select(XYDimName, {
+        mode: "within-polygon",
+        polygon
+      });
+    }
+
+    case "graph lasso cancel":
+    case "graph brush cancel":
+    case "graph lasso deselect":
+    case "graph brush deselect": {
+      return state.select(XYDimName, { mode: "all" });
+    }
+
+    case "continuous metadata histogram start":
+    case "continuous metadata histogram brush":
+    case "continuous metadata histogram cancel":
+    case "continuous metadata histogram end": {
+      const name = makeContinuousDimensionName(
+        action.continuousNamespace,
+        action.selection
+      );
+      // action.selection: metadata name being selected
+      // action.range: filter range, or null if deselected
+      if (!action.range) {
+        return state.select(name, { mode: "all" });
+      }
+      const [lo, hi] = action.range;
+      const newState = state.select(name, { mode: "range", lo, hi });
+      return newState;
+    }
+
+    case "categorical metadata filter select":
+    case "categorical metadata filter deselect": {
+      const { categoricalSelection } = nextSharedState;
+      const cat = categoricalSelection[action.metadataField];
+      const { categoryValues, categoryValueSelected } = cat;
+      const values = categoryValues.filter((v, i) => categoryValueSelected[i]);
+      return state.select(obsAnnoDimensionName(action.metadataField), {
+        mode: "exact",
+        values
+      });
+    }
+
+    case "categorical metadata filter none of these": {
+      return state.select(obsAnnoDimensionName(action.metadataField), {
+        mode: "none"
+      });
+    }
+
+    case "categorical metadata filter all of these": {
+      return state.select(obsAnnoDimensionName(action.metadataField), {
+        mode: "all"
+      });
+    }
+
+    default: {
+      return state;
+    }
+  }
+};
+
+/*
+  IMPORTANT: the system assumes that crossfilter.data() will point at the
+  same value as world.obsAnnotations.  For actions handled in this reducer,
+  make sure that this remains true.
+
+  This wrapper performs only this function.
+*/
+const CrossfilterReducer = (
+  state,
+  action,
+  nextSharedState,
+  prevSharedState
+) => {
+  const nextState = CrossfilterReducerBase(
+    state,
+    action,
+    nextSharedState,
+    prevSharedState
+  );
+  if (!nextState || nextState.all() === nextSharedState.world.obsAnnotations) {
+    return nextState;
+  }
+  return nextState.setData(nextSharedState.world.obsAnnotations);
+};
+
+export default CrossfilterReducer;

client/src/reducers/graphSelection.js

@@ -0,0 +1,61 @@
+const GraphSelection = (
+  state = {
+    tool: "lasso", // what selection tool mode (lasso, brush, ...)
+    selection: { mode: "all" } // current selection, which is tool specific
+  },
+  action
+) => {
+  switch (action.type) {
+    case "set clip quantiles":
+    case "reset World to eq Universe":
+    case "set layout choice": {
+      return {
+        ...state,
+        selection: {
+          mode: "all"
+        }
+      };
+    }
+
+    case "graph brush end":
+    case "graph brush change": {
+      const { brushCoords } = action;
+      return {
+        ...state,
+        selection: {
+          mode: "within-rect",
+          brushCoords
+        }
+      };
+    }
+
+    case "graph lasso end": {
+      const { polygon } = action;
+      return {
+        ...state,
+        selection: {
+          mode: "within-polygon",
+          polygon
+        }
+      };
+    }
+
+    case "graph lasso cancel":
+    case "graph brush cancel":
+    case "graph lasso deselect":
+    case "graph brush deselect": {
+      return {
+        ...state,
+        selection: {
+          mode: "all"
+        }
+      };
+    }
+
+    default: {
+      return state;
+    }
+  }
+};
+
+export default GraphSelection;

client/src/reducers/index.js

@@ -1,27 +1,62 @@
-// jshint esversion: 6
-import { combineReducers, createStore, applyMiddleware } from "redux";
+import { createStore, applyMiddleware } from "redux";
 import thunk from "redux-thunk";
-import { composeWithDevTools } from "redux-devtools-extension";
-import updateURLMiddleware from "../middleware/updateURLMiddleware";
-import updateCellColors from "../middleware/updateCellColors";
 
+import cascadeReducers from "./cascade";
+import undoable from "./undoable";
 import config from "./config";
+import universe from "./universe";
+import world from "./world";
+import categoricalSelection from "./categoricalSelection";
+import continuousSelection from "./continuousSelection";
+import graphSelection from "./graphSelection";
+import crossfilter from "./crossfilter";
+import colors from "./colors";
 import differential from "./differential";
+import layoutChoice from "./layoutChoice";
 import responsive from "./responsive";
 import controls from "./controls";
+import resetCache from "./resetCache";
+import centroidLabel from "./centroidLabel";
+import annotations from "./annotations";
+import autosave from "./autosave";
 
-const Reducer = combineReducers({
-  config,
-  responsive,
-  controls,
-  differential
-});
+import undoableConfig from "./undoableConfig";
 
-const store = createStore(
-  Reducer,
-  composeWithDevTools(
-    applyMiddleware(thunk, updateURLMiddleware, updateCellColors)
-  )
+const Reducer = undoable(
+    cascadeReducers([
+        ["config", config],
+        ["universe", universe],
+        ["world", world],
+        ["annotations", annotations],
+        ["layoutChoice", layoutChoice],
+        ["categoricalSelection", categoricalSelection],
+        ["continuousSelection", continuousSelection],
+        ["graphSelection", graphSelection],
+        ["crossfilter", crossfilter],
+        ["colors", colors],
+        ["controls", controls],
+        ["differential", differential],
+        ["responsive", responsive],
+        ["centroidLabel", centroidLabel],
+        ["autosave", autosave],
+        ["resetCache", resetCache]
+    ]),
+    [
+        "universe",
+        "world",
+        "categoricalSelection",
+        "continuousSelection",
+        "graphSelection",
+        "crossfilter",
+        "colors",
+        "controls",
+        "differential",
+        "layoutChoice",
+        "annotations"
+    ],
+    undoableConfig
 );
 
+const store = createStore(Reducer, applyMiddleware(thunk));
+
 export default store;

client/src/reducers/layoutChoice.js

@@ -0,0 +1,50 @@
+/*
+we have a UI heuristic to pick the default layout, based on assumptions
+about commonly used names.  Preferentially, pick in the following order:
+
+  1. "umap"
+  2. "tsne"
+  3. "pca"
+  4. give up, use the first available
+*/
+function bestDefaultLayout(layouts) {
+  const preferredNames = ["umap", "tsne", "pca"];
+  const idx = preferredNames.findIndex(name => layouts.indexOf(name) !== -1);
+  if (idx !== -1) return preferredNames[idx];
+  return layouts[0];
+}
+
+const LayoutChoice = (
+  state = {
+    available: [], // all available choices
+    current: undefined, // name of the current layout, eg, 'umap'
+    currentDimNames: [] // dimension name
+  },
+  action,
+  nextSharedState
+) => {
+  switch (action.type) {
+    case "initial data load complete (universe exists)":
+    case "reset World to eq Universe": {
+      // set default to default
+      const { schema } = nextSharedState.world;
+      const available = schema.layout.obs.map(v => v.name);
+      const current = bestDefaultLayout(available);
+      const currentDimNames = schema.layout.obsByName[current].dims;
+      return { available, current, currentDimNames };
+    }
+
+    case "set layout choice": {
+      const { schema } = nextSharedState.world;
+      const current = action.layoutChoice;
+      const currentDimNames = schema.layout.obsByName[current].dims;
+      return { ...state, current, currentDimNames };
+    }
+
+    default: {
+      return state;
+    }
+  }
+};
+
+export default LayoutChoice;

client/src/reducers/resetCache.js

@@ -0,0 +1,30 @@
+/*
+Reducer which caches derived state to be used in a reset or other 
+recomputation.  Add stuff here you want stashed at init time (or whenever),
+for later use.
+
+Currently this only caches the baseline (full universe) crossfilter, 
+which improves Reset UI performance.
+*/
+const ResetCacheReducer = (
+  state = {
+    crossfilter: null
+  },
+  action,
+  nextSharedState
+) => {
+  switch (action.type) {
+    case "initial data load complete (universe exists)": {
+      const { crossfilter } = nextSharedState;
+      return {
+        ...state,
+        crossfilter
+      };
+    }
+    default: {
+      return state;
+    }
+  }
+};
+
+export default ResetCacheReducer;

client/src/reducers/undoable.js

@@ -0,0 +1,287 @@
+/*
+A redo/undo meta reducer for Redux.   Designed to work well with the cascadeReducer().
+
+Requires three parameters:
+* reducer - a reducer, which MUST return an object as state.
+* undoableKeys - an array of object keys (strings).   If any of these keys
+  are in the object/state returned by the reducer, they will be treated as
+  state to be made "undoable".
+* options - an optional object, which may contain the following parameters:
+    * historyLimit: max number of historical states to remember (aka max undo depth)
+    * actionFilter: filter function, (state, action, filterState) => value.
+      See below for details.
+    * debug: if truish, will print helpful log messages about history manipulation
+
+This meta reducer accepts three actions types:
+* @@undoable/undo - move back in history
+* @@undoable/redo - move forward in history
+* @@undoable/clear - clear history
+
+---
+
+Action filter - controls the undoable reducer side-effects.  If not
+specified, the action filter defaults to "save", ie, pushes a redo
+point upon each action.
+
+The action filter callback has access to the current action, the entire
+undoable reducer state, and any state it wants to manage ("filterState").
+This filter state will be passed to each action filter call, and any
+value returned (via @@undoable/filterState field described below) will be
+MERGED into the current filter state.
+
+An object must be returned (the "undoable action"), indicating desired
+history state processing.  The undoable action object contents, by key:
+
+  @@undoable/filterAction: required.   Can be one of:
+      "skip" - reduce the current action, but no other side effects.
+        Same as returning false.
+      "clear" - reduce the current action, and clear history state.
+      "save" - push the previous state onto the history stack (ie,
+        before reducing the action)
+      "stashPending" - reduce action, save state as pending.  Does not
+        not commit it to history.  Along with cancelPending and applyPending,
+        can be used to delay commit of history (eg, for multi-action
+        groupings, asynch operations, etc).
+      "cancelPending" - reduce action, cancel any pending state save.
+      "applyPending" - commit any pending state to the history stack,
+        then reduce action.
+
+  @@undoable/filterState: optional.  If this value is set, it will be
+    MERGED into the current filter state. The value and semantics of any
+    filter state are entirely at the discretion of the action filter.
+
+*/
+import fromEntries from "../util/fromEntries";
+
+const historyKeyPrefix = "@@undoable/";
+const pastKey = `${historyKeyPrefix}past`;
+const futureKey = `${historyKeyPrefix}future`;
+const filterStateKey = `${historyKeyPrefix}filterState`;
+const filterActionKey = `${historyKeyPrefix}filterAction`;
+const pendingKey = `${historyKeyPrefix}pending`;
+const defaultHistoryLimit = -100;
+
+const Undoable = (reducer, undoableKeys, options = {}) => {
+  const { debug } = options;
+  let { historyLimit } = options;
+  if (!historyLimit) historyLimit = defaultHistoryLimit;
+  if (historyLimit > 0) historyLimit = -historyLimit;
+  const actionFilter =
+    options.actionFilter || (() => ({ [filterActionKey]: "save" }));
+
+  if (!Array.isArray(undoableKeys) || undoableKeys.length === 0)
+    throw new Error("undoable keys array must be specified");
+  const undoableKeysSet = new Set(undoableKeys);
+
+  /*
+  Undo the current to previous history
+  */
+  function undo(currentState) {
+    const past = currentState[pastKey];
+    const future = currentState[futureKey];
+    if (past.length === 0) return currentState;
+    const currentUndoableState = Object.entries(currentState).filter(kv =>
+      undoableKeysSet.has(kv[0])
+    );
+    const newPast = [...past];
+    const newState = newPast.pop();
+    const newFuture = push(future, currentUndoableState);
+    const nextState = {
+      ...currentState,
+      ...fromEntries(newState),
+      [pastKey]: newPast,
+      [futureKey]: newFuture,
+      [pendingKey]: null
+    };
+    return nextState;
+  }
+
+  /*
+  Replay future, previously undone.
+  */
+  function redo(currentState) {
+    const past = currentState[pastKey] || [];
+    const future = currentState[futureKey] || [];
+    if (future.length === 0) return currentState;
+    const currentUndoableState = Object.entries(currentState).filter(kv =>
+      undoableKeysSet.has(kv[0])
+    );
+    const newFuture = [...future];
+    const newState = newFuture.pop();
+    const newPast = push(past, currentUndoableState);
+    const nextState = {
+      ...currentState,
+      ...fromEntries(newState),
+      [pastKey]: newPast,
+      [futureKey]: newFuture,
+      [pendingKey]: null
+    };
+    return nextState;
+  }
+
+  /*
+  Clear the history state.  No side-effects on current state.
+  */
+  function clear(currentState) {
+    return {
+      ...currentState,
+      [pastKey]: [],
+      [futureKey]: [],
+      [filterStateKey]: {},
+      [pendingKey]: null
+    };
+  }
+
+  /*
+  Reduce current action, with no history side-effects
+  */
+  function skip(currentState, action, filterState) {
+    const past = currentState[pastKey] || [];
+    const pending = currentState[pendingKey];
+    const res = reducer(currentState, action);
+    return {
+      ...res,
+      [pastKey]: past,
+      [futureKey]: [],
+      [filterStateKey]: filterState,
+      [pendingKey]: pending
+    };
+  }
+
+  /*
+  Save current state in the history, then reduce action.
+  */
+  function save(currentState, action, filterState) {
+    const past = currentState[pastKey] || [];
+    const currentUndoableState = Object.entries(currentState).filter(kv =>
+      undoableKeysSet.has(kv[0])
+    );
+    const res = reducer(currentState, action);
+    const newPast = push(past, currentUndoableState, historyLimit);
+    const nextState = {
+      ...res,
+      [pastKey]: newPast,
+      [futureKey]: [],
+      [filterStateKey]: filterState,
+      [pendingKey]: null
+    };
+    return nextState;
+  }
+
+  /*
+  Save current state as pending history change.  No other side effects.
+  */
+  function stashPending(currentState) {
+    const currentUndoableState = Object.entries(currentState).filter(kv =>
+      undoableKeysSet.has(kv[0])
+    );
+    return {
+      ...currentState,
+      [pendingKey]: currentUndoableState
+    };
+  }
+
+  /*
+  Cancel pending history state change.  No other side effects.
+  */
+  function cancelPending(currentState) {
+    return {
+      ...currentState,
+      [pendingKey]: null
+    };
+  }
+
+  /*
+  Push pending state onto the history stack
+  */
+  function applyPending(currentState) {
+    const past = currentState[pastKey] || [];
+    const pendingState = currentState[pendingKey];
+    const newPast = push(past, pendingState, historyLimit);
+    const nextState = {
+      ...currentState,
+      [pastKey]: newPast,
+      [futureKey]: [],
+      [pendingKey]: null
+    };
+    return nextState;
+  }
+
+  return (
+    currentState = {
+      [pastKey]: [],
+      [futureKey]: [],
+      [filterStateKey]: {},
+      [pendingKey]: null
+    },
+    action
+  ) => {
+    const aType = action.type;
+    switch (aType) {
+      case "@@undoable/undo": {
+        return undo(currentState, action);
+      }
+
+      case "@@undoable/redo": {
+        return redo(currentState, action);
+      }
+
+      case "@@undoable/clear": {
+        return clear(currentState, action);
+      }
+
+      default: {
+        const currentFilterState = currentState[filterStateKey];
+        const actionFilterResp = actionFilter(
+          currentState,
+          action,
+          currentFilterState
+        );
+        const {
+          [filterActionKey]: filterAction,
+          [filterStateKey]: filterStateUpdate
+        } = actionFilterResp;
+        const nextFilterState = { ...currentFilterState, ...filterStateUpdate };
+
+        switch (filterAction) {
+          case "clear":
+            if (debug) console.log("---- CLEAR HISTO", action.type);
+            return clear(skip(currentState, action, nextFilterState));
+
+          case "save":
+            if (debug) console.log("---- SAVE HISTO", action.type);
+            return save(currentState, action, nextFilterState);
+
+          case "stashPending":
+            if (debug) console.log("---- STASH PENDING", action.type);
+            return skip(stashPending(currentState), action, nextFilterState);
+
+          case "cancelPending":
+            if (debug) console.log("---- CANCEL PENDING", action.type);
+            return skip(cancelPending(currentState), action, nextFilterState);
+
+          case "applyPending":
+            if (debug) console.log("---- APPLY PENDING", action.type);
+            return skip(applyPending(currentState), action, nextFilterState);
+
+          case "skip":
+          default:
+            return skip(currentState, action, nextFilterState);
+        }
+      }
+    }
+  };
+};
+
+function push(arr, val, limit = undefined) {
+  /*
+  functional array push, with a max length limit to the new array.
+  Like Array.push, except it returns new array and discards as needed
+  to enforce the length limit.
+  */
+  const narr = arr.slice(limit);
+  narr.push(val);
+  return narr;
+}
+
+export default Undoable;

client/src/reducers/undoableConfig.js

@@ -0,0 +1,271 @@
+import StateMachine from "../util/statemachine";
+import createFsmTransitions from "./undoableFsm";
+
+const actionKey = "@@undoable/filterAction";
+const stateKey = "@@undoable/filterState";
+
+/*
+these actions will not affect history
+*/
+const skipOnActions = new Set([
+  "url changed",
+  "interface reset started",
+  "initial data load start",
+  "configuration load complete",
+  "increment graph render counter",
+  "window resize",
+  "user reset start",
+  "reset colorscale",
+
+  "graph brush change",
+  "continuous metadata histogram brush",
+
+  "expression load start",
+  "expression load success",
+  "expression load error",
+
+  "request user defined gene started",
+  "request user defined gene success",
+  "clear all user defined genes",
+
+  "get single gene expression for coloring started",
+  "get single gene expression for coloring error",
+
+  "category value mouse hover start",
+  "category value mouse hover end"
+]);
+
+/*
+identical, repeated occurances of these action types will be debounced.
+Entire action must be identical (all keys).
+*/
+const debounceOnActions = new Set([
+  "color by categorical metadata",
+  "color by continuous metadata",
+  "color by expression"
+]);
+
+/*
+history will be cleared when these actions occur
+*/
+const clearOnActions = new Set([
+  "initial data load complete (universe exists)",
+  "reset World to eq Universe",
+  "initial data load error",
+  "user reset end"
+]);
+
+/*
+An immediate history save will be done for these
+*/
+const saveOnActions = new Set([
+  "categorical metadata filter select",
+  "categorical metadata filter deselect",
+  "categorical metadata filter all of these",
+  "categorical metadata filter none of these",
+
+  "color by categorical metadata",
+  "color by continuous metadata",
+  "color by expression",
+
+  "set scatterplot x",
+  "set scatterplot y",
+
+  "store current cell selection as differential set 1",
+  "store current cell selection as differential set 2",
+
+  "set World to current selection",
+  "set clip quantiles",
+
+  "set layout choice",
+  "change graph interaction mode",
+
+  // user editable annotations
+  "annotation: create category",
+  "annotation: add new label to category",
+  "annotation: delete category",
+  "annotation: label edited",
+  "annotation: label current cell selection",
+  "annotation: delete label"
+]);
+
+/**
+StateMachine - processing complex action handling - see FSM graph for
+actual structure, in undoableFsm.js
+**/
+
+/*
+Default FSM actions.  Used to side-effect transitions in the graph.
+See graph definition for the transitions that use each.
+
+Signature:  (fsm, transition, reducerState, reducerAction) => undoableAction
+*/
+const stashPending = fsm => ({
+  [actionKey]: "stashPending",
+  [stateKey]: { fsm }
+});
+const cancelPending = () => ({
+  [actionKey]: "cancelPending",
+  [stateKey]: { fsm: null }
+});
+const applyPending = () => ({
+  [actionKey]: "applyPending",
+  [stateKey]: { fsm: null }
+});
+const skip = (fsm, transition) => ({
+  [actionKey]: "skip",
+  [stateKey]: { fsm: transition.to !== "done" ? fsm : null }
+});
+const clear = () => ({ [actionKey]: "clear", [stateKey]: { fsm: null } });
+const save = (fsm, transition) => ({
+  [actionKey]: "save",
+  [stateKey]: { fsm: transition.to !== "done" ? fsm : null }
+});
+
+/*
+Error handler for state transitions that are unexpected.  Called by
+StateMachine when it doesn't know what to do.
+
+Signature:  (fsm, event, from) => undoableAction
+*/
+const onFsmError = (fsm, event, from) => {
+  console.error(`FSM error [event: "${event}", state: "${from}"]`, fsm);
+  // In production, try to recover gracefully if we have unexpected state
+  return clear(fsm);
+};
+
+/*
+Definition of the transition graph mapping action types to history side effects.
+*/
+const fsmTransitions = createFsmTransitions(
+  stashPending,
+  cancelPending,
+  applyPending,
+  skip,
+  clear,
+  save
+);
+/* State machine we clone whenever we need to run it */
+const seedFsm = new StateMachine("init", fsmTransitions, onFsmError);
+
+/*
+See undoable.js for description action filter interface description.
+
+Basic approach:
+  * trivial handlers for skip, clear & save cases to keep config simple.
+  * only implement complex state machines where absolutely required (eg,
+    multi-event seleciton and the like)
+*/
+const actionFilter = debug => (state, action, prevFilterState) => {
+  const actionType = action.type;
+  const filterState = {
+    ...prevFilterState,
+    prevAction: action
+  };
+  if (skipOnActions.has(actionType)) {
+    return { [actionKey]: "skip", [stateKey]: filterState };
+  }
+  if (
+    debounceOnActions.has(actionType) &&
+    shallowObjectEq(action, prevFilterState.prevAction)
+  ) {
+    return { [actionKey]: "skip", [stateKey]: filterState };
+  }
+  if (clearOnActions.has(actionType)) {
+    return { [actionKey]: "clear", [stateKey]: filterState };
+  }
+  if (saveOnActions.has(actionType)) {
+    return { [actionKey]: "save", [stateKey]: filterState };
+  }
+
+  /*
+    Else, something more complex OR unknown to us....
+    */
+  if (seedFsm.events.has(actionType)) {
+    let { fsm } = filterState;
+    if (!fsm) {
+      /* no active FSM, so create one in init state */
+      fsm = seedFsm.clone("init");
+    }
+    return fsm.next(action.type, { state, action });
+  }
+
+  /* else, we have no idea what this is - skip it */
+  if (debug) console.log("**** ACTION FILTER EVENT HANDLER MISS", actionType);
+  return { [actionKey]: "skip", [stateKey]: filterState };
+};
+
+/*
+return true if objA and objB are ===, OR if:
+  - are both objects and not null
+  - have same own properties
+  - all values are strict equal (===)
+*/
+function shallowObjectEq(objA, objB) {
+  if (objA === objB) return true;
+  if (!objA || !objB) return false;
+  if (!shallowArrayEq(Object.keys(objA), Object.keys(objB))) return false;
+  if (!shallowArrayEq(Object.values(objA), Object.values(objB))) return false;
+  return true;
+}
+
+/*
+return true if arrA and arrB contain the same strict-equal values,
+in the same order.
+*/
+function shallowArrayEq(arrA, arrB) {
+  if (arrA.length !== arrB.length) return false;
+  for (let i = 0, l = arrA.length; i < l; i += 1) {
+    if (arrA[i] !== arrB[i]) return false;
+  }
+  return true;
+}
+
+/* configuration for the undoable meta reducer */
+const debug = false; // set truish for undoble debugging
+const undoableConfig = {
+  debug,
+  historyLimit: 50, // maximum history size
+  actionFilter: actionFilter(debug)
+};
+
+/*
+this code is strictly for sanity checking configuration, and is only
+enabled when we are debugging the undoable configuration (ie, debug === true).
+*/
+if (debug) {
+  /*
+  Confirm no intersection between the various trivial rejection action filters
+  */
+  if (
+    new Set([...skipOnActions].filter(x => clearOnActions.has(x))).size > 0 ||
+    new Set([...skipOnActions].filter(x => saveOnActions.has(x))).size > 0 ||
+    new Set([...clearOnActions].filter(x => saveOnActions.has(x))).size > 0
+  ) {
+    console.error(
+      "Undoable misconfiguration - action filters have redundant events"
+    );
+  }
+
+  /*
+  Confirm that no FSM events are blocked by a trivial rejection filter.
+  If this occurs, the FSM can't ever see the events needed to process
+  state transitions.
+  */
+  const trivialFilters = new Set([
+    ...skipOnActions,
+    ...clearOnActions,
+    ...saveOnActions
+  ]);
+  const trivialOverlapWithFsm = new Set(
+    [...trivialFilters].filter(x => seedFsm.events.has(x))
+  );
+  if (trivialOverlapWithFsm.size > 0) {
+    console.error(
+      "Undoable misconfiguration - trivival action filter blocking FSM filter",
+      [...trivialOverlapWithFsm]
+    );
+  }
+}
+
+export default undoableConfig;

client/src/reducers/undoableFsm.js

@@ -0,0 +1,206 @@
+/*
+State transition graph for complex action/history interactions.
+
+Assumed configuration from undoableConfig:
+    * By convention, "init" is used as the start state for all, and "done"
+      as the final state.
+    * Unexpected states will result in an error, plus a clear and cancelPending
+      side-effect.
+
+TODO: is is possible there is a more concise format for this, as it is
+a fairly repetitive pattern.
+
+These events are largely one of two types:
+a) async operations or multi-event options that should only be committed
+upon some success criteria, otherwise cancelled.
+
+b) compound actions that should be collapsed into a single history change.
+
+*/
+
+const createFsmTransitions = (
+  stashPending,
+  cancelPending,
+  applyPending,
+  skip,
+  clear,
+  save
+) => {
+  return [
+    /* graph selection brushing */
+    {
+      event: "graph brush start",
+      from: "init",
+      to: "graph brush in progress",
+      action: stashPending
+    },
+    {
+      event: "graph brush cancel",
+      from: "graph brush in progress",
+      to: "done",
+      action: applyPending
+    },
+    {
+      event: "graph brush deselect",
+      from: "graph brush in progress",
+      to: "done",
+      /* if current selection is all, cancelPending.  Else, applyPending */
+      action: (fsm, transition, data) =>
+        data.state.graphSelection.selection.mode === "all"
+          ? cancelPending()
+          : applyPending()
+    },
+    {
+      event: "graph brush end",
+      from: "graph brush in progress",
+      to: "done",
+      action: applyPending
+    },
+
+    /* graph selection lasso */
+    {
+      event: "graph lasso start",
+      from: "init",
+      to: "graph lasso in progress",
+      action: stashPending
+    },
+    {
+      event: "graph lasso cancel",
+      from: "graph lasso in progress",
+      to: "done",
+      action: applyPending
+    },
+    {
+      event: "graph lasso deselect",
+      from: "graph lasso in progress",
+      to: "done",
+      /* if current selection is all, cancelPending.  Else, applyPending */
+      action: (fsm, transition, data) =>
+        data.state.graphSelection.selection.mode === "all"
+          ? cancelPending()
+          : applyPending()
+    },
+    {
+      event: "graph lasso end",
+      from: "graph lasso in progress",
+      to: "done",
+      action: applyPending
+    },
+
+    /* Continuous metadata histogram brush selection */
+    {
+      event: "continuous metadata histogram start",
+      from: "init",
+      to: "continuous histo select in progress",
+      action: stashPending
+    },
+    {
+      event: "continuous metadata histogram cancel",
+      from: "continuous histo select in progress",
+      to: "done",
+      action: cancelPending
+    },
+    {
+      event: "continuous metadata histogram end",
+      from: "continuous histo select in progress",
+      to: "done",
+      action: applyPending
+    },
+
+    /* Single gene request by user */
+    {
+      event: "single user defined gene start",
+      from: "init",
+      to: "single user gene request in progress",
+      action: stashPending
+    },
+    {
+      event: "request user defined gene error",
+      from: "single user gene request in progress",
+      to: "single user gene error in progress",
+      action: skip
+    },
+    {
+      event: "single user defined gene error",
+      from: "single user gene error in progress",
+      to: "done",
+      action: cancelPending
+    },
+    {
+      event: "single user defined gene complete",
+      from: "single user gene request in progress",
+      to: "done",
+      action: applyPending
+    },
+
+    /* Bulk gene request by user */
+    {
+      event: "bulk user defined gene start",
+      from: "init",
+      to: "bulk user gene request in progress",
+      action: stashPending
+    },
+    {
+      event: "request user defined gene error",
+      from: "bulk user gene request in progress",
+      to: "bulk user gene request error in progress",
+      action: skip
+    },
+    {
+      event: "bulk user defined gene error",
+      from: "bulk user gene request error in progress",
+      to: "done",
+      action: cancelPending
+    },
+    {
+      event: "bulk user defined gene complete",
+      from: "bulk user gene request in progress",
+      to: "done",
+      action: applyPending
+    },
+
+    /* Compute Differential Expression button user action */
+    {
+      event: "request differential expression started",
+      from: "init",
+      to: "diffexp in progress",
+      action: stashPending
+    },
+    {
+      event: "request user defined gene error",
+      from: "diffexp in progress",
+      to: "done",
+      action: cancelPending
+    },
+    {
+      event: "request differential expression success",
+      from: "diffexp in progress",
+      to: "done",
+      action: applyPending
+    },
+
+    /* Clear Differential Expression button user action */
+    {
+      event: "clear differential expression",
+      from: "init",
+      to: "CDE Button in progress",
+      action: stashPending
+    },
+    {
+      event: "clear scatterplot",
+      from: "CDE Button in progress",
+      to: "done",
+      action: applyPending
+    },
+
+    /* clear scatter plot button (eg, on scatterplot view) */
+    {
+      event: "clear scatterplot",
+      from: "init",
+      to: "done",
+      action: save
+    }
+  ];
+};
+
+export default createFsmTransitions;

client/src/reducers/universe.js

@@ -0,0 +1,226 @@
+import { unassignedCategoryLabel } from "../globals";
+import {
+  World,
+  ControlsHelpers as CH,
+  AnnotationsHelpers as AH
+} from "../util/stateManager";
+
+const Universe = (state = null, action, nextSharedState, prevSharedState) => {
+  switch (action.type) {
+    case "initial data load complete (universe exists)": {
+      const { universe } = action;
+      return universe;
+    }
+
+    case "expression load success": {
+      let { varData } = state;
+
+      // Lazy load new expression data into the varData dataframe, if
+      // not already present.
+      //
+      Object.entries(action.expressionData).forEach(([key, val]) => {
+        // If not already in universe.varData, save entire expression column
+        if (!varData.hasCol(key)) {
+          varData = varData.withCol(key, val);
+        }
+      });
+
+      // Prune size of varData "cache" if getting out of hand....
+      //
+      const { userDefinedGenes, diffexpGenes } = prevSharedState;
+      const allTheGenesWeNeed = [
+        ...new Set(
+          userDefinedGenes,
+          diffexpGenes,
+          Object.keys(action.expressionData)
+        )
+      ];
+      varData = CH.pruneVarDataCache(varData, allTheGenesWeNeed);
+      return { ...state, varData };
+    }
+
+    case "annotation: create category": {
+      /* create a new annotation category, with all values set to 'unassigned' */
+      const name = action.data;
+      const { categoryToDuplicate } = action;
+
+      /* name must be a string,  non-zero length */
+      if (typeof name !== "string" || name.length === 0)
+        throw new Error("user annotations require string name");
+
+      /* ensure the name isn't already in use! */
+      if (state.obsAnnotations.hasCol(name))
+        throw new Error("name collision on annotation category create");
+
+      /* ensure the duplicate col exists */
+      if (
+        categoryToDuplicate &&
+        !state.obsAnnotations.hasCol(categoryToDuplicate)
+      )
+        throw new Error("categoryToDuplicate does not exist");
+
+      let schema;
+      let data;
+      if (categoryToDuplicate) {
+        /* duplicate the named annotation */
+        schema = AH.dupObsAnnoSchema(state.schema, categoryToDuplicate, name, {
+          writable: true
+        });
+        /* if we are duplicating a non-writable annotation, it may not have an unassigned category */
+        const s = schema.annotations.obsByName[categoryToDuplicate];
+        if (s.categories.indexOf(unassignedCategoryLabel) === -1) {
+          s.categories = s.categories.concat(unassignedCategoryLabel);
+        }
+        data = state.obsAnnotations.col(categoryToDuplicate).asArray();
+      } else {
+        /* else, all are unassined */
+        const categories = [unassignedCategoryLabel];
+        schema = AH.addObsAnnoSchema(state.schema, name, {
+          name,
+          categories,
+          type: "categorical",
+          writable: true
+        });
+        data = new Array(state.nObs).fill(unassignedCategoryLabel);
+      }
+
+      const obsAnnotations = state.obsAnnotations.withCol(name, data);
+      return { ...state, obsAnnotations, schema };
+    }
+
+    case "annotation: category edited": {
+      /* change the name of an obs annotation category */
+      const name = action.metadataField;
+      const newName = action.newCategoryText;
+      if (!AH.isUserAnnotation(state, name))
+        throw new Error("unable to edit read-only annotation");
+      if (typeof newName !== "string" || newName.length === 0)
+        throw new Error("user annotations require string name");
+
+      const colSchema = {
+        ...state.schema.annotations.obsByName[name],
+        name: newName
+      };
+      const schema = AH.addObsAnnoSchema(
+        AH.removeObsAnnoSchema(state.schema, name),
+        newName,
+        colSchema
+      );
+      const obsAnnotations = state.obsAnnotations.renameCol(name, newName);
+      return { ...state, schema, obsAnnotations };
+    }
+
+    case "annotation: delete category": {
+      /* delete annotation category from schema and obsAnnotations */
+      const name = action.metadataField;
+      if (!AH.isUserAnnotation(state, name))
+        throw new Error("unable to delete read-only annotation");
+
+      const schema = AH.removeObsAnnoSchema(state.schema, name);
+      const obsAnnotations = state.obsAnnotations.dropCol(name);
+      return { ...state, schema, obsAnnotations };
+    }
+
+    case "annotation: add new label to category": {
+      const annotationName = action.metadataField;
+      const newLabelName = action.newLabelText;
+      if (!AH.isUserAnnotation(state, annotationName))
+        throw new Error("unable to modify read-only annotation");
+      if (typeof newLabelName !== "string" || newLabelName.length === 0)
+        throw new Error(
+          "user annotations require a non-zero length string name"
+        );
+
+      /* add the new label to the annotation */
+      const schema = AH.addObsAnnoCategory(
+        state.schema,
+        annotationName,
+        newLabelName
+      );
+      return { ...state, schema };
+    }
+
+    case "annotation: label edited": {
+      const annotationName = action.metadataField;
+      const oldLabelName = action.label;
+      const newLabelName = action.editedLabel;
+      if (!AH.isUserAnnotation(state, annotationName))
+        throw new Error("unable to modify read-only annotation");
+      if (typeof newLabelName !== "string" || newLabelName.length === 0)
+        throw new Error(
+          "user annotations require a non-zero length string name"
+        );
+
+      /* remove old label, add new label */
+      const schema = AH.addObsAnnoCategory(
+        AH.removeObsAnnoCategory(state.schema, annotationName, oldLabelName),
+        annotationName,
+        newLabelName
+      );
+
+      /* change all values in obsAnnotation */
+      const obsAnnotations = AH.setLabelByValue(
+        state.obsAnnotations,
+        annotationName,
+        oldLabelName,
+        newLabelName
+      );
+
+      return { ...state, schema, obsAnnotations };
+    }
+
+    case "annotation: delete label": {
+      /* delete the label from the annotation, and set all cells with this value to unassigned */
+      const annotationName = action.metadataField;
+      const labelName = action.label;
+      if (!AH.isUserAnnotation(state, annotationName))
+        throw new Error("unable to modify read-only annotation");
+      if (labelName === unassignedCategoryLabel)
+        throw new Error("may not remove the unassigned label");
+
+      /* remove the category from the schema */
+      const schema = AH.removeObsAnnoCategory(
+        state.schema,
+        annotationName,
+        labelName
+      );
+
+      /* set all values to unassigned in obsAnnotations */
+      const obsAnnotations = AH.setLabelByValue(
+        state.obsAnnotations,
+        annotationName,
+        labelName,
+        unassignedCategoryLabel
+      );
+
+      return { ...state, schema, obsAnnotations };
+    }
+
+    case "annotation: label current cell selection": {
+      const { metadataField, label } = action;
+      const { world, crossfilter } = prevSharedState;
+
+      /*
+      selection state is relative to world.  We need to convert it
+      to a mask for Universe before applying it.
+      */
+      const worldMask = crossfilter.allSelectedMask();
+      const mask = World.worldEqUniverse(world, state)
+        ? worldMask
+        : AH.worldToUniverseMask(worldMask, world.obsAnnotations, state.nObs);
+      const obsAnnotations = AH.setLabelByMask(
+        state.obsAnnotations,
+        metadataField,
+        mask,
+        label
+      );
+      return { ...state, obsAnnotations };
+    }
+
+    default: {
+      return state;
+    }
+  }
+};
+
+export default Universe;

client/src/reducers/world.js

@@ -0,0 +1,272 @@
+import { unassignedCategoryLabel } from "../globals";
+import {
+  World,
+  ControlsHelpers as CH,
+  AnnotationsHelpers as AH
+} from "../util/stateManager";
+import clip from "../util/clip";
+import quantile from "../util/quantile";
+
+/*
+important note: much of this code assumes that wriable (user) annotations
+will NOT contain scalar data (ie, will only contain categorical labelled
+data), and therefore will never need to be clipped.  Put another way, it
+assumes that for these annotations, the clipped & unclipped data is equal.
+
+If we ever start allowing user editable scalar data, this assumption will
+need to be revisited.
+*/
+
+const WorldReducer = (
+  state = null,
+  action,
+  nextSharedState,
+  prevSharedState
+) => {
+  switch (action.type) {
+    case "initial data load complete (universe exists)":
+    case "reset World to eq Universe": {
+      const { universe } = nextSharedState;
+      const world = World.createWorldFromEntireUniverse(universe);
+      return world;
+    }
+
+    case "set World to current selection": {
+      /* Set viewable world to be the currently selected data */
+      const world = World.createWorldBySelection(
+        action.universe,
+        action.world,
+        action.crossfilter
+      );
+      return world;
+    }
+
+    case "set clip quantiles": {
+      const world = World.createWorldWithNewClip(
+        prevSharedState.universe,
+        state,
+        prevSharedState.crossfilter,
+        action.clipQuantiles
+      );
+      return world;
+    }
+
+    case "expression load success": {
+      const { universe } = nextSharedState;
+      const universeVarData = universe.varData;
+      let unclippedVarData = state.unclipped.varData;
+
+      // Lazy load new expression data into the unclipped varData dataframe, if
+      // not already present.
+      //
+      Object.entries(action.expressionData).forEach(([key, val]) => {
+        // If not already in world.varData, save sliced expression column
+        if (!unclippedVarData.hasCol(key)) {
+          // Slice if world !== universe, else just use whole column.
+          // Use the obsAnnotation index as the cut key, as we keep
+          // all world dataframes in sync.
+          let worldValSlice = val;
+          if (!World.worldEqUniverse(state, universe)) {
+            worldValSlice = universeVarData
+              .subset(state.obsAnnotations.rowIndex.keys(), [key], null)
+              .icol(0)
+              .asArray();
+          }
+
+          // Now build world's varData dataframe
+          unclippedVarData = unclippedVarData.withCol(
+            key,
+            worldValSlice,
+            state.obsAnnotations.rowIndex
+          );
+        }
+      });
+
+      // Prune size of varData unclipped dataframe if getting out of hand....
+      //
+      const { userDefinedGenes, diffexpGenes } = prevSharedState;
+      const allTheGenesWeNeed = [
+        ...new Set(
+          userDefinedGenes,
+          diffexpGenes,
+          Object.keys(action.expressionData)
+        )
+      ];
+      unclippedVarData = CH.pruneVarDataCache(
+        unclippedVarData,
+        allTheGenesWeNeed
+      );
+
+      // at this point, we have the unclipped data in unclippedVarData.
+      // Now create clipped.
+      //   - Drop columns no longer needed
+      //   - Add new columns
+      //
+      let clippedVarData = state.varData;
+      const keysToDrop = clippedVarData.colIndex
+        .keys()
+        .filter(k => !unclippedVarData.hasCol(k));
+      const keysToAdd = unclippedVarData.colIndex
+        .keys()
+        .filter(k => !clippedVarData.hasCol(k));
+      keysToDrop.forEach(k => {
+        clippedVarData = clippedVarData.dropCol(k);
+      });
+      keysToAdd.forEach(k => {
+        const data = unclippedVarData.col(k).asArray();
+        const q = [state.clipQuantiles.min, state.clipQuantiles.max];
+        const [qMinVal, qMaxVal] = quantile(q, data);
+        const clippedData = clip(data, qMinVal, qMaxVal, Number.NaN);
+        clippedVarData = clippedVarData.withCol(
+          k,
+          clippedData,
+          state.obsAnnotations.rowIndex
+        );
+      });
+
+      return {
+        ...state,
+        varData: clippedVarData,
+        unclipped: {
+          ...state.unclipped,
+          varData: unclippedVarData
+        }
+      };
+    }
+
+    case "annotation: create category": {
+      const name = action.data;
+      const { universe } = nextSharedState;
+      const { schema } = universe;
+
+      /*
+      if world !== universe, we have to subset the newly created annotation,
+      else, just use it as is.
+      */
+      let newAnnotation = null;
+      if (!World.worldEqUniverse(state, universe)) {
+        newAnnotation = universe.obsAnnotations
+          .subset(state.obsAnnotations.rowIndex.keys(), [name], null)
+          .icol(0)
+          .asArray();
+      } else {
+        newAnnotation = universe.obsAnnotations.col(name).asArray();
+      }
+      const obsAnnotations = state.obsAnnotations.withCol(
+        name,
+        newAnnotation,
+        state.obsAnnotations.rowIndex
+      );
+      const unclipped = {
+        ...state.unclipped,
+        obsAnnotations: state.unclipped.obsAnnotations.withCol(
+          name,
+          newAnnotation,
+          state.unclipped.obsAnnotations.rowIndex
+        )
+      };
+      return { ...state, schema, obsAnnotations, unclipped };
+    }
+
+    case "annotation: category edited": {
+      /* change the name of an obs annotation */
+      const name = action.metadataField;
+      const newName = action.newCategoryText;
+      const { schema } = nextSharedState.universe;
+      const obsAnnotations = state.obsAnnotations.renameCol(name, newName);
+      const unclipped = {
+        ...state.unclipped,
+        obsAnnotations: state.unclipped.obsAnnotations.renameCol(name, newName)
+      };
+      return { ...state, schema, obsAnnotations, unclipped };
+    }
+
+    case "annotation: delete category": {
+      /* remove a category from obs annotation */
+      const { schema } = nextSharedState.universe;
+      const name = action.metadataField;
+      const obsAnnotations = state.obsAnnotations.dropCol(name);
+      const unclipped = {
+        ...state.unclipped,
+        obsAnnotations: state.unclipped.obsAnnotations.dropCol(name)
+      };
+      return { ...state, schema, obsAnnotations, unclipped };
+    }
+
+    case "annotation: add new label to category": {
+      /* add a new label to the schema - schema updated by universe reducer, we just need to note it */
+      const { schema } = nextSharedState.universe;
+      return { ...state, schema };
+    }
+
+    case "annotation: label edited": {
+      const { schema } = nextSharedState.universe;
+      const { metadataField } = action;
+      const oldLabelName = action.label;
+      const newLabelName = action.editedLabel;
+
+      /* set all values to to new label */
+      const unclipped = {
+        ...state.unclipped,
+        obsAnnotations: AH.setLabelByValue(
+          state.unclipped.obsAnnotations,
+          metadataField,
+          oldLabelName,
+          newLabelName
+        )
+      };
+      const obsAnnotations = state.obsAnnotations.replaceColData(
+        metadataField,
+        unclipped.obsAnnotations.col(metadataField).asArray()
+      );
+      return { ...state, schema, obsAnnotations, unclipped };
+    }
+
+    case "annotation: delete label": {
+      const { schema } = nextSharedState.universe;
+      const { label, metadataField } = action;
+
+      /* set all values to unassigned in obsAnnotations */
+      const unclipped = {
+        ...state.unclipped,
+        obsAnnotations: AH.setLabelByValue(
+          state.unclipped.obsAnnotations,
+          metadataField,
+          label,
+          unassignedCategoryLabel
+        )
+      };
+      const obsAnnotations = state.obsAnnotations.replaceColData(
+        metadataField,
+        unclipped.obsAnnotations.col(metadataField).asArray()
+      );
+      return { ...state, schema, obsAnnotations, unclipped };
+    }
+
+    case "annotation: label current cell selection": {
+      const { metadataField, label } = action;
+      const { crossfilter } = prevSharedState;
+      const mask = crossfilter.allSelectedMask();
+      const unclipped = {
+        ...state.unclipped,
+        obsAnnotations: AH.setLabelByMask(
+          state.unclipped.obsAnnotations,
+          metadataField,
+          mask,
+          label
+        )
+      };
+      const obsAnnotations = state.obsAnnotations.replaceColData(
+        metadataField,
+        unclipped.obsAnnotations.col(metadataField).asArray()
+      );
+      return { ...state, obsAnnotations, unclipped };
+    }
+
+    default: {
+      return state;
+    }
+  }
+};
+
+export default WorldReducer;

client/src/util/camera.js

@@ -1,67 +1,181 @@
-// jshint esversion: 6
-const createCamera = require("orbit-camera");
-const createScroll = require("scroll-speed");
-const mp = require("mouse-position");
-const mb = require("mouse-pressed");
-const key = require("key-pressed");
+import { vec2, mat3 } from "gl-matrix";
+
+const EPSILON = 0.000001;
 
-const panSpeed = 0.4;
 const scaleSpeed = 0.5;
-const scaleMax = 3;
-// const scaleMin = 1.15
-const scaleMin = 1.03;
+const scaleMax = 3.0;
+const scaleMin = 0.5;
+const panBound = 0.8;
 
-function attachCamera(canvas, opts) {
-  opts = opts || {};
-  opts.pan = opts.pan !== false;
-  opts.scale = opts.scale !== false;
-  opts.rotate = opts.rotate !== false;
+// private
+const scratch0 = new Float32Array(16);
+const scratch1 = new Float32Array(16);
 
-  const scroll = createScroll(canvas, opts.scale);
-  const mbut = mb(canvas, opts.rotate);
-  const mpos = mp(canvas);
-  const camera = createCamera([0, 0, 1], [0, 0, -1], [0, 1, 0]);
+function clamp(val, rng) {
+  return Math.max(Math.min(val, rng[1]), rng[0]);
+}
 
-  camera.tick = tick;
+class Camera {
+  constructor(canvas) {
+    this.prevEvent = {
+      clientX: 0,
+      clientY: 0,
+      type: 0
+    };
+    this.canvas = canvas;
+    this.viewMatrix = mat3.create();
+    this.viewMatrixInv = mat3.create();
+  }
 
-  return camera;
+  view() {
+    return this.viewMatrix;
+  }
 
-  function tick() {
-    const ctrl = key("<control>") || key("<alt>");
-    const alt = key("<shift>");
-    const { height, width } = canvas;
+  invView() {
+    return this.viewMatrixInv;
+  }
 
-    if (opts.rotate && mbut.left && ctrl && !alt) {
-      camera.rotate(
-        [mpos.x / width - 0.5, mpos.y / height - 0.5],
-        [mpos.prevX / width - 0.5, mpos.prevY / height - 0.5]
-      );
+  distance() {
+    return this.viewMatrix[0];
+  }
+
+  pan(dx, dy) {
+    const m = this.viewMatrix;
+    const dyRange = [
+      -panBound - (m[7] + 1) / m[4],
+      panBound - (m[7] - 1) / m[4]
+    ];
+    const dxRange = [
+      -panBound - (m[6] + 1) / m[0],
+      panBound - (m[6] - 1) / m[0]
+    ];
+
+    const dxClamped = clamp(dx, dxRange);
+    const dyClamped = clamp(dy, dyRange);
+    if (Math.abs(dxClamped) <= EPSILON && Math.abs(dyClamped) <= EPSILON)
+      return;
+
+    mat3.translate(m, m, [dxClamped, dyClamped]);
+    mat3.invert(this.viewMatrixInv, m);
+  }
+
+  zoomAt(d, x = 0, y = 0) {
+    /*
+    Camera zoom at [x,y]
+    */
+    const m = this.viewMatrix;
+    const bounds = [-panBound, panBound];
+    x = clamp(x, bounds);
+    y = clamp(y, bounds);
+
+    const dClamped = clamp(d * m[0], [scaleMin, scaleMax]) / m[0];
+    if (Math.abs(1 - dClamped) <= EPSILON) return; // noop request
+
+    mat3.translate(m, m, [x, y]);
+    mat3.scale(m, m, [dClamped, dClamped]);
+    mat3.translate(m, m, [-x, -y]);
+
+    mat3.invert(this.viewMatrixInv, m);
+  }
+
+  /*
+  Event handling
+  */
+
+  flush(e) {
+    this.prevEvent.type = e.type;
+    this.prevEvent.clientX = e.clientX;
+    this.prevEvent.clientY = e.clientY;
+  }
+
+  localPosition(target, canvasX, canvasY, projectionInvTF) {
+    /*
+    Convert mouse position to local
+    */
+    const { height, width } = target;
+    const targetRect = target.getBoundingClientRect();
+    canvasX -= targetRect.left;
+    canvasY -= targetRect.top;
+
+    const pos = vec2.fromValues(
+      2 * (canvasX / width) - 1,
+      -2 * (canvasY / height) + 1
+    );
+    if (projectionInvTF) {
+      vec2.transformMat3(pos, pos, projectionInvTF);
     }
+    vec2.transformMat3(pos, pos, this.invView());
+    return pos;
+  }
 
-    if ((opts.pan && mbut.right) || (mbut.left && !ctrl && !alt)) {
-      camera.pan([
-        ((panSpeed * (mpos[0] - mpos.prev[0])) / width) * camera.distance,
-        ((panSpeed * (mpos[1] - mpos.prev[1])) / height) * camera.distance
-      ]);
+  mousePan(e, projectionTF) {
+    const projectionInvTF = mat3.invert(scratch0, projectionTF);
+    const pos = this.localPosition(
+      this.canvas,
+      e.clientX,
+      e.clientY,
+      projectionInvTF
+    );
+    const prev = this.localPosition(
+      this.canvas,
+      this.prevEvent.clientX,
+      this.prevEvent.clientY,
+      projectionInvTF
+    );
+
+    const delta = vec2.sub(scratch1, pos, prev);
+    this.pan(delta[0], delta[1]);
+    return true;
+  }
+
+  wheelZoom(e, projectionTF) {
+    const { height } = this.canvas;
+    const { deltaY, deltaMode, clientX, clientY } = e;
+    const scale = scaleSpeed * (deltaMode === 1 ? 12 : 1) * (deltaY || 0);
+
+    const projectionInvTF = mat3.invert(scratch0, projectionTF);
+    const pos = this.localPosition(
+      this.canvas,
+      clientX,
+      clientY,
+      projectionInvTF
+    );
+    this.zoomAt(1 / Math.exp(scale / height), pos[0], pos[1]);
+    return true;
+  }
+
+  handleEvent(e, projectionTF) {
+    /*
+    process the event, and return true if camera view changed
+    */
+    let viewChanged = false;
+    switch (e.type) {
+      case "mousemove": {
+        /* eslint-disable no-bitwise */
+        if (e.buttons & 0x1) {
+          viewChanged = this.mousePan(e, projectionTF);
+        }
+        /* eslint-enable no-bitwise */
+        this.flush(e);
+        break;
+      }
+
+      case "wheel": {
+        viewChanged = this.wheelZoom(e, projectionTF);
+        this.flush(e);
+        break;
+      }
+
+      default:
+        // noop
+        break;
     }
-
-    if (opts.scale && scroll[1]) {
-      camera.distance *= Math.exp((scroll[1] * scaleSpeed) / height);
-    }
-
-    if (opts.scale && (mbut.middle || (mbut.left && !ctrl && alt))) {
-      const d = mpos.y - mpos.prevY;
-      if (!d) return;
-
-      camera.distance *= Math.exp(d / height);
-    }
-
-    if (camera.distance > scaleMax) camera.distance = scaleMax;
-    if (camera.distance < scaleMin) camera.distance = scaleMin;
-
-    scroll.flush();
-    mpos.flush();
+    return viewChanged;
   }
 }
 
+function attachCamera(canvas) {
+  return new Camera(canvas);
+}
+
 export default attachCamera;

client/src/util/centroid.js

@@ -0,0 +1,58 @@
+import quantile from "./quantile";
+
+/*
+  Centroid coordinate calculation
+*/
+const calcMeanCentroid = (world, annoName, annoValue, layoutDimNames) => {
+  const centroid = { x: 0, y: 0, size: 0 };
+  const annoArray = world.obsAnnotations.col(annoName).asArray();
+  const layoutXArray = world.obsLayout.col(layoutDimNames[0]).asArray();
+  const layoutYArray = world.obsLayout.col(layoutDimNames[1]).asArray();
+
+  for (let i = 0, len = annoArray.length; i < len; i += 1) {
+    if (annoArray[i] === annoValue) {
+      centroid.x += layoutXArray[i];
+      centroid.y += layoutYArray[i];
+      centroid.size += 1;
+    }
+  }
+
+  if (centroid[2] !== 0) {
+    centroid.x /= centroid.size;
+    centroid.y /= centroid.size;
+  }
+
+  return [centroid.x, centroid.y];
+};
+
+const calcMedianCentroid = (world, annoName, annoValue, layoutDimNames) => {
+  const centroidX = [];
+  const centroidY = [];
+  let hasFinite = false;
+
+  const annoArray = world.obsAnnotations.col(annoName).asArray();
+  const layoutXArray = world.obsLayout.col(layoutDimNames[0]).asArray();
+  const layoutYArray = world.obsLayout.col(layoutDimNames[1]).asArray();
+
+  for (let i = 0, len = annoArray.length; i < len; i += 1) {
+    if (annoArray[i] === annoValue) {
+      hasFinite =
+        Number.isFinite(layoutXArray[i]) || Number.isFinite(layoutYArray[i])
+          ? true
+          : hasFinite;
+      centroidX.push(layoutXArray[i]);
+      centroidY.push(layoutYArray[i]);
+    }
+  }
+
+  if (hasFinite) {
+    const medianX = quantile([0.5], Float64Array.from(centroidX));
+    const medianY = quantile([0.5], Float64Array.from(centroidY));
+
+    return [medianX, medianY];
+  }
+
+  return null;
+};
+
+export default calcMedianCentroid;

client/src/util/clip.js

@@ -0,0 +1,25 @@
+/*
+clip - clip all values in a Array or TypedArray, IN PLACE.
+
+Values in array are clipped if less than `lower` or greater than `upper`.
+
+If `setTo` is undefined, values less than `lower` will be set to `lower`,
+and values greater than `upper` will be set to `upper`.  
+
+If `setTo` is not undefined, values outside the [lower, upper] range will be set to 
+`setTo`.
+
+*/
+export default function clip(arr, lower, upper, setTo) {
+	const lowerSet = setTo === undefined ? lower : setTo;
+	const upperSet = setTo === undefined ? upper : setTo;
+	for (let i = 0, l = arr.length; i < l; i += 1) {
+		const v = arr[i];
+		if (v < lower) {
+			arr[i] = lowerSet;
+		} else if (v > upper) {
+			arr[i] = upperSet;
+		}
+	}
+	return arr;
+}

client/src/util/dataframe/dataframe.js

@@ -1,8 +1,19 @@
 import { IdentityInt32Index, isLabelIndex } from "./labelIndex";
 // weird cross-dependency that we should clean up someday...
-import { sort } from "../typedCrossfilter/sort";
-import { isTypedArray, isArrayOrTypedArray, callOnceLazy } from "./util";
+import { sortArray } from "../typedCrossfilter/sort";
+import {
+  isTypedArray,
+  isArrayOrTypedArray,
+  callOnceLazy,
+  memoize
+} from "./util";
 import { summarizeContinuous, summarizeCategorical } from "./summarize";
+import {
+  histogramCategorical,
+  hashCategorical,
+  histogramContinuous,
+  hashContinuous
+} from "./histogram";
 
 /*
 Dataframe is an immutable 2D matrix similiar to Python Pandas Dataframe,
@@ -59,11 +70,28 @@ Dataframe
 **/
 
 class Dataframe {
+  /**
+  memoization helpers.
+  **/
+  static __DataframeId__ = 0;
+
+  static __getId() {
+    const id = Dataframe.__DataframeId__;
+    Dataframe.__DataframeId__ += 1;
+    return id;
+  }
+
   /**
   Constructors & factories
   **/
 
-  constructor(dims, columnarData, rowIndex = null, colIndex = null) {
+  constructor(
+    dims,
+    columnarData,
+    rowIndex = null,
+    colIndex = null,
+    __columnsAccessor = [] // private interface
+  ) {
     /*
     The base constructor is relatively hard to use - as an alternative,
     see factory methods and clone/slice, below.
@@ -74,6 +102,9 @@ class Dataframe {
         or TypedArray of length nRows.
       * rowIndex/colIndex - null (create default index using offsets as key),
         or a caller-provided index.
+      * __columnsAccessor - private interface, do not specify.  Used internally
+        to improve caching of column accessors when possible (eg, clone(), 
+        dropCol(), withCol()).
     All columns and indices must have appropriate dimensionality.
     */
     const [nRows, nCols] = dims;
@@ -93,8 +124,10 @@ class Dataframe {
     this.length = nRows; // convenience accessor for row dimension
     this.rowIndex = rowIndex;
     this.colIndex = colIndex;
+    this.__id = Dataframe.__getId();
 
-    this.__compile();
+    this.__compile(__columnsAccessor);
+    Object.freeze(this);
   }
 
   static __errorChecks(dims, columnarData, rowIndex, colIndex) {
@@ -135,98 +168,129 @@ class Dataframe {
     }
   }
 
-  __compile() {
+  static __compileColumn(column, getRowByOffset, getRowByLabel) {
+    /*
+      Each column accessor is a function which will lookup data by
+      index (ie, is equivalent to dataframe.get(row, col), where 'col'
+      is fixed.
+
+      In addition, each column accessor has several functions:
+
+      asArray() -- return the entire column as a native Array or TypedArray.
+        Crucially, this native array only supports label indexing.
+        Example:
+          const arr = df.col('a').asArray();
+
+      has(rlabel) -- return boolean indicating of the row label
+        is contained within the column.  Example:
+          const isInColumn = df.col('a').includes(99)
+        For the default offset indexing, this is identical to:
+          const isInColumn = (99 > 0) && (99 < df.nRows);
+
+      ihas(roffset) -- same as has(), but accepts a row offset
+        instead of a row label.
+
+      indexOf(value) -- return the label (not offset) of the first instance of
+        'value' in the column.  If you want the offset, just use the builtin JS
+        indexOf() function, available on both Array and TypedArray.
+
+      iget(offset) -- return the value at 'offset'
+
+      ... and more ...
+
+    */
+    const { length } = column;
+    const __id = Dataframe.__getId();
+
+    /* get value by row label */
+    const get = function get(rlabel) {
+      return column[getRowByOffset(rlabel)];
+    };
+
+    /* get value by row offset */
+    const iget = function iget(roffset) {
+      return column[roffset];
+    };
+
+    /* full column array access */
+    const asArray = function asArray() {
+      return column;
+    };
+
+    /* test for row label inclusion in column */
+    const has = function has(rlabel) {
+      const offset = getRowByOffset(rlabel);
+      return offset >= 0 && offset < length;
+    };
+
+    const ihas = function ihas(offset) {
+      return offset >= 0 && offset < length;
+    };
+
+    /*
+    return first label (index) at which the value is found in this column,
+    or undefined if not found.
+
+    NOTE: not found return is DIFFERENT than the default Array.indexOf as
+    -1 is a plausible Dataframe row/col label.
+    */
+    const indexOf = function indexOf(value) {
+      const offset = column.indexOf(value);
+      if (offset === -1) {
+        return undefined;
+      }
+      return getRowByLabel(offset);
+    };
+
+    /*
+    Summarize the column data. Lazy eval;
+    */
+    const summarize = callOnceLazy(() =>
+      isTypedArray(column)
+        ? summarizeContinuous(column)
+        : summarizeCategorical(column)
+    );
+
+    /*
+    Create histogram bins for this column.  Memoized.
+    */
+    if (isTypedArray(column)) {
+      const mFn = memoize(histogramContinuous, hashContinuous);
+      get.histogram = (bins, domain, by) => mFn(get, bins, domain, by);
+    } else {
+      const mFn = memoize(histogramCategorical, hashCategorical);
+      get.histogram = by => mFn(get, by);
+    }
+
+    get.summarize = summarize;
+    get.asArray = asArray;
+    get.has = has;
+    get.ihas = ihas;
+    get.indexOf = indexOf;
+    get.iget = iget;
+    get.__id = __id;
+
+    Object.freeze(get);
+    return get;
+  }
+
+  __compile(accessors) {
     /*
     Compile data accessors for each column.
 
-    Each column accessor is a function which will lookup data by
-    index (ie, is equivalent to dataframe.get(row, col), where 'col'
-    is fixed.
-
-    In addition, each column accessor has several functions:
-
-    asArray() -- return the entire column as a native Array or TypedArray.
-      Crucially, this native array only supports label indexing.
-      Example:
-        const arr = df.col('a').asArray();
-
-    has(rlabel) -- return boolean indicating of the row label
-      is contained within the column.  Example:
-        const isInColumn = df.col('a').includes(99)
-      For the default offset indexing, this is identical to:
-        const isInColumn = (99 > 0) && (99 < df.nRows);
-
-    ihas(roffset) -- same as has(), but accepts a row offset
-      instead of a row label.
-
-    indexOf(value) -- return the label (not offset) of the first instance of
-      'value' in the column.  If you want the offset, just use the builtin JS
-      indexOf() function, available on both Array and TypedArray.
-
-    iget(offset) -- return the value at 'offset'
-
+    Use an existing accessor if provided, else compile a new one.
     */
-    const { getOffset, getLabel } = this.rowIndex;
-    this.__columnsAccessor = this.__columns.map(column => {
-      const { length } = column;
-
-      /* get value by row label */
-      const get = function get(rlabel) {
-        return column[getOffset(rlabel)];
-      };
-
-      /* get value by row offset */
-      const iget = function iget(roffset) {
-        return column[roffset];
-      };
-
-      /* full column array access */
-      const asArray = function asArray() {
-        return column;
-      };
-
-      /* test for row label inclusion in column */
-      const has = function has(rlabel) {
-        const offset = getOffset(rlabel);
-        return offset >= 0 && offset < length;
-      };
-
-      const ihas = function ihas(offset) {
-        return offset >= 0 && offset < length;
-      };
-
-      /*
-      return first label (index) at which the value is found in this column,
-      or undefined if not found.
-
-      NOTE: not found return is DIFFERENT than the default Array.indexOf as
-      -1 is a plausible Dataframe row/col label.
-      */
-      const indexOf = function indexOf(value) {
-        const offset = column.indexOf(value);
-        if (offset === -1) {
-          return undefined;
-        }
-        return getLabel(offset);
-      };
-
-      /*
-      Summarize the column data. Lazy eval;
-      */
-      const summarize = callOnceLazy(() =>
-        isTypedArray(column)
-          ? summarizeContinuous(column)
-          : summarizeCategorical(column)
-      );
-
-      get.summarize = summarize;
-      get.asArray = asArray;
-      get.has = has;
-      get.ihas = ihas;
-      get.indexOf = indexOf;
-      get.iget = iget;
-      return get;
+    const {
+      getOffset: getRowByOffset,
+      getLabel: getRowByLabel
+    } = this.rowIndex;
+    this.__columnsAccessor = this.__columns.map((column, idx) => {
+      if (accessors[idx]) {
+        return accessors[idx];
+      }
+      return Dataframe.__compileColumn(column, getRowByOffset, getRowByLabel);
     });
+    Object.freeze(this.__columnsAccessor);
   }
 
   clone() {
@@ -237,7 +301,8 @@ class Dataframe {
       this.dims,
       [...this.__columns],
       this.rowIndex,
-      this.colIndex
+      this.colIndex,
+      [...this.__columnsAccessor]
     );
   }
 
@@ -273,7 +338,39 @@ class Dataframe {
     const columns = [...this.__columns];
     columns.push(colData);
     const colIndex = this.colIndex.withLabel(label);
-    return new this.constructor(dims, columns, rowIndex, colIndex);
+    const columnsAccessor = [...this.__columnsAccessor];
+    return new this.constructor(
+      dims,
+      columns,
+      rowIndex,
+      colIndex,
+      columnsAccessor
+    );
+  }
+
+  withColsFrom(dataframe) {
+    /*
+    return a new dataframe containing all columns from both `this` and the
+    provided dataframe.
+
+    The row index from `this` will be used.  Both dataframes must have identical
+    dimensionality, and no overlapping columns labels.
+     */
+    const dims = [this.dims[0], this.dims[1] + dataframe.dims[1]];
+    const { rowIndex } = this;
+    const columns = [...this.__columns, ...dataframe.__columns];
+    const colIndex = this.colIndex.withLabels(dataframe.colIndex.keys());
+    const columnsAccessor = [
+      ...this.__columnsAccessor,
+      ...dataframe.__columnsAccessor
+    ];
+    return new this.constructor(
+      dims,
+      columns,
+      rowIndex,
+      colIndex,
+      columnsAccessor
+    );
   }
 
   dropCol(label) {
@@ -281,13 +378,78 @@ class Dataframe {
     Create a new dataframe, omitting one columns.
 
     const newDf = df.dropCol("colors");
+
+    Corner case to manage: if dropping the last column, return an empty dataframe.
     */
+    if (!this.hasCol(label)) {
+      throw new RangeError(`unknown label: ${label}`);
+    }
+
+    /* 
+    Corner case to manage: if dropping the last column, return an empty dataframe. 
+    */
+    if (this.dims[1] === 1) {
+      return Dataframe.empty();
+    }
+
     const dims = [this.dims[0], this.dims[1] - 1];
     const coffset = this.colIndex.getOffset(label);
     const columns = [...this.__columns];
     columns.splice(coffset, 1);
     const colIndex = this.colIndex.dropLabel(label);
-    return new this.constructor(dims, columns, this.rowIndex, colIndex);
+    const columnsAccessor = [...this.__columnsAccessor];
+    columnsAccessor.splice(coffset, 1);
+    return new this.constructor(
+      dims,
+      columns,
+      this.rowIndex,
+      colIndex,
+      columnsAccessor
+    );
+  }
+
+  renameCol(oldLabel, newLabel) {
+    /*
+    Accelerator for dropping a column and then adding it again with a new label
+    */
+    const coffset = this.colIndex.getOffset(oldLabel);
+    const colIndex = this.colIndex.dropLabel(oldLabel).withLabel(newLabel);
+
+    const columns = [...this.__columns];
+    columns.push(columns[coffset]);
+    columns.splice(coffset, 1);
+
+    const columnsAccessor = [...this.__columnsAccessor];
+    columnsAccessor.push(columnsAccessor[coffset]);
+    columnsAccessor.splice(coffset, 1);
+
+    return new this.constructor(
+      this.dims,
+      columns,
+      this.rowIndex,
+      colIndex,
+      columnsAccessor
+    );
+  }
+
+  replaceColData(label, newColData) {
+    /*
+    Accelerator for dropping a column then adding it again with same
+    label and different values.
+    */
+    const coffset = this.colIndex.getOffset(label);
+    const columns = [...this.__columns];
+    columns[coffset] = newColData;
+    const columnsAccessor = [...this.__columnsAccessor];
+    columnsAccessor[coffset] = null;
+
+    return new this.constructor(
+      this.dims,
+      columns,
+      this.rowIndex,
+      this.colIndex,
+      columnsAccessor
+    );
   }
 
   static empty(rowIndex = null, colIndex = null) {
@@ -316,7 +478,7 @@ class Dataframe {
       if (!offsets) {
         return [null, null];
       }
-      const sortedOffsets = sort(offsets);
+      const sortedOffsets = sortArray(offsets);
       const sortedLabels = new Array(sortedOffsets.length);
       for (let i = 0, l = sortedOffsets.length; i < l; i += 1) {
         sortedLabels[i] = index.getLabel(sortedOffsets[i]);
@@ -366,6 +528,8 @@ class Dataframe {
         return newCol;
       });
     }
+
+    if (dims[0] === 0 || dims[1] === 0) return Dataframe.empty();
     return new Dataframe(dims, columns, rowIndex, colIndex);
   }
 
@@ -449,6 +613,11 @@ class Dataframe {
   Data access with row/col.
   **/
 
+  columns() {
+    /* return all column accessors as an array, in offset order */
+    return [...this.__columnsAccessor];
+  }
+
   col(columnLabel) {
     /*
     Return accessor bound to a column.  Allows random row access
@@ -543,14 +712,34 @@ class Dataframe {
   /****
   Functional (map/reduce/etc) data access
 
-  XXX: not yet implemented, as there is no clear use case.   Can easily
+  TODO: most are not yet implemented, as there is no clear use case.   Can easily
   add these as useful.
   ****/
 
+  mapColumns(callback) {
+    /*
+    map all columns in the dataframe, returning a new dataframe comprised of the
+    return values, with the same index as the original dataframe.
+
+    callback MUST not modify the column, but instead return a mutated copy.
+    */
+    const columns = this.__columns.map(callback);
+    const columnsAccessor = columns.map((c, idx) =>
+      this.__columns[idx] === c ? this.__columnsAccessor[idx] : undefined
+    );
+    return new this.constructor(
+      this.dims,
+      columns,
+      this.rowIndex,
+      this.colIndex,
+      columnsAccessor
+    );
+  }
+
   /*
   Map & reduce of column or row
 
-  XXX TODO remainder of map/reduce functions:  mapCol, mapRow, reduceRow, ...
+  TODO remainder of map/reduce functions:  mapCol, mapRow, reduceRow, ...
   */
   /* comment out until we have a use for this
 

client/src/util/dataframe/histogram.js

@@ -0,0 +1,135 @@
+/*
+Dataframe histogram
+*/
+import { isTypedArray } from "./util";
+
+function _histogramContinuous(column, bins, min, max) {
+  const valBins = new Array(bins).fill(0);
+  if (!column) {
+    return valBins;
+  }
+  const binWidth = (max - min) / bins;
+  const colArray = column.asArray();
+  for (let r = 0, len = colArray.length; r < len; r += 1) {
+    const val = colArray[r];
+    if (val <= max && val >= min) {
+      // ensure test excludes NaN values
+      const valBin = Math.min(Math.floor((val - min) / binWidth), bins - 1);
+      valBins[valBin] += 1;
+    }
+  }
+  return valBins;
+}
+
+function _histogramContinuousBy(column, bins, min, max, by) {
+  const byMap = new Map();
+  if (!column || !by) {
+    return byMap;
+  }
+  const binWidth = (max - min) / bins;
+  const byArray = by.asArray();
+  const colArray = column.asArray();
+  for (let r = 0, len = colArray.length; r < len; r += 1) {
+    const byBin = byArray[r];
+    let valBins = byMap.get(byBin);
+    if (valBins === undefined) {
+      valBins = new Array(bins).fill(0);
+      byMap.set(byBin, valBins);
+    }
+    const val = colArray[r];
+    if (val <= max && val >= min) {
+      // ensure test excludes NaN values
+      const valBin = Math.min(Math.floor((val - min) / binWidth), bins - 1);
+      valBins[valBin] += 1;
+    }
+  }
+  return byMap;
+}
+
+function _histogramCategorical(column) {
+  const valMap = new Map();
+  if (!column) {
+    return valMap;
+  }
+  const colArray = column.asArray();
+  for (let r = 0, len = colArray.length; r < len; r += 1) {
+    const valBin = colArray[r];
+    let curCount = valMap.get(valBin);
+    if (curCount === undefined) {
+      curCount = 0;
+    }
+    valMap.set(valBin, curCount + 1);
+  }
+  return valMap;
+}
+
+function _histogramCategoricalBy(column, by) {
+  const byMap = new Map();
+  if (!column || !by) {
+    return byMap;
+  }
+  const byArray = by.asArray();
+  const colArray = column.asArray();
+  for (let r = 0, len = colArray.length; r < len; r += 1) {
+    const byBin = byArray[r];
+    let valMap = byMap.get(byBin);
+    if (valMap === undefined) {
+      valMap = new Map();
+      byMap.set(byBin, valMap);
+    }
+    const valBin = colArray[r];
+    let curCount = valMap.get(valBin);
+    if (curCount === undefined) {
+      curCount = 0;
+    }
+    valMap.set(valBin, curCount + 1);
+  }
+  return byMap;
+}
+
+/*
+Count category occupancy.  Optional group-by category.
+*/
+export function histogramCategorical(column, by) {
+  if (by && isTypedArray(by)) {
+    throw new Error("Group by column must be categorical");
+  }
+  return by
+    ? _histogramCategoricalBy(column, by)
+    : _histogramCategorical(column);
+}
+
+/*
+Memoization hash for histogramCategorical()
+*/
+export function hashCategorical(column, by) {
+  if (by) {
+    return `${column.__id}:${by.__id}`;
+  }
+  return `${column.__id}:`;
+}
+
+/*
+Bin counts for continuous/scalar values, with optional group-by category.
+Values outside domain are ignored.
+*/
+export function histogramContinuous(column, bins = 40, domain = [0, 1], by) {
+  if (by && isTypedArray(by)) {
+    throw new Error("Group by column must be categorical");
+  }
+  const [min, max] = domain;
+  return by
+    ? _histogramContinuousBy(column, bins, min, max, by)
+    : _histogramContinuous(column, bins, min, max);
+}
+
+/*
+Memoization hash for histogramContinuous
+*/
+export function hashContinuous(column, bins = "", domain = [0, 0], by) {
+  const [min, max] = domain;
+  if (by) {
+    return `${column.__id}:${bins}:${min}:${max}:${by.__id}`;
+  }
+  return `${column.__id}::${bins}:${min}:${max}`;
+}

client/src/util/dataframe/labelIndex.js

@@ -3,6 +3,8 @@ Label indexing - map a label to & from an integer offset.  See Dataframe
 for how this is used.
 **/
 
+import { rangeFill as fillRange } from "../range";
+
 /*
 Private utility functions
 */
@@ -21,14 +23,6 @@ function extent(tarr) {
   return [min, max];
 }
 
-function fillRange(arr, start = 0) {
-  const larr = arr;
-  for (let i = 0, l = larr.length; i < l; i += 1) {
-    larr[i] = i + start;
-  }
-  return larr;
-}
-
 /* eslint-disable class-methods-use-this */
 class IdentityInt32Index {
   /*
@@ -86,6 +80,10 @@ class IdentityInt32Index {
     return this.__promote([...this.keys(), label]);
   }
 
+  withLabels(labels) {
+    return this.__promote([...this.keys(), ...labels]);
+  }
+
   dropLabel(label) {
     if (label === this.maxOffset - 1) {
       return new IdentityInt32Index(label);
@@ -169,6 +167,10 @@ class DenseInt32Index {
     return this.__promote([...this.keys(), label]);
   }
 
+  withLabels(labels) {
+    return this.__promote([...this.keys(), ...labels]);
+  }
+
   dropLabel(label) {
     const labelArray = [...this.keys()];
     labelArray.splice(labelArray.indexOf(label), 1);
@@ -193,6 +195,11 @@ class KeyIndex {
       index.set(v, i);
     });
 
+    if (index.size !== rindex.length) {
+      /* if true, there was a duplicate in the keys */
+      throw new Error("duplicate label provided to KeyIndex");
+    }
+
     this.index = index;
     this.rindex = rindex;
     this.__compile();
@@ -224,6 +231,10 @@ class KeyIndex {
     return new KeyIndex([...this.rindex, label]);
   }
 
+  withLabels(labels) {
+    return new KeyIndex([...this.rindex, ...labels]);
+  }
+
   dropLabel(label) {
     const idx = this.rindex.indexOf(label);
     const labelArray = [...this.rindex];