bump version

They finally fixed their cloud pickle issue

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.9.1
+current_version = 0.2.4
 
 [bumpversion:file:setup.py]
 search = version="{current_version}"

.dockerignore

@@ -1,5 +0,0 @@
-bin
-client
-dist
-docs
-server

.gitignore

@@ -27,9 +27,6 @@ server/app/web/templates/index\.html
 .ipynb_checkpoints
 *.ipynb
 
-# cefpython
-error.log
-
 # misc
 .DS_Store
 npm-debug.log
@@ -38,7 +35,3 @@ npm-debug.log
 __pycache__
 *.DS_Store*
 data
-
-# Jekyll
-docs/_site/
-docs/Gemfile.lock

.travis.yml

@@ -1,32 +1,20 @@
 language: python
-dist: xenial
+python:
-sudo: required
+  - "3.6"
 node_js:
-  - 8
+  - "8"
 cache:
-  - pip
+  pip: true
-  - npm
 install:
   - set -eo pipefail
   - pip install flake8
-  - make pydist
+  - ./bin/build-client
-  - make install-dist
+  - pip install -e .
   - pip install -r server/requirements-dev.txt
-
+script:
-jobs:
+  - set -eo pipefail
-  include:
+  - flake8 server/app/
-  - name: "Branch Tests 3.7"
+  - flake8 server/cli/
-    python: "3.7"
+  - npm run --prefix client/ build
-    script: ./travis-build.sh
+  - npm run --prefix client/ test
-  - name: "Branch Tests 3.6"
+  - pytest -s server/test
-    python: "3.6"
-    script: ./travis-build.sh
-  - name: "Docker Build"
-    install: skip
-    python: "3.6"
-    script: docker build .
-  - name: "Smoke Tests"
-    python: "3.6"
-    if: branch = master AND type = cron
-    script:
-      - npm run --prefix client/ smoke-test

Dockerfile

@@ -1,10 +0,0 @@
-FROM ubuntu:bionic
-
-ENV LC_ALL=C.UTF-8
-ENV LANG=C.UTF-8
-
-RUN apt-get update && \
-    apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev && \
-    pip3 install cellxgene
-
-ENTRYPOINT ["cellxgene"]

README.md

@@ -2,59 +2,164 @@
 
 > an interactive explorer for single-cell transcriptomics data
 
-_cellxgene_ (pronounced "sell-by-jean") is an interactive data explorer for single-cell transcriptomics datasets, such as those coming from the [Human Cell Atlas](https://humancellatlas.org). Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data, and to demonstrate general, scalable, and reusable patterns for scientific data visualization.
+`cellxgene` is an interactive data explorer for single-cell transcriptomics datasets, such as those coming from the [Human Cell Atlas](https://humancellatlas.org). Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data, and to demonstrate general, scalable, and reusable patterns for scientific data visualization.
 
 <img src="https://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/docs/cellxgene-demo-1.gif" width="200" height="200" hspace="30"><img src="https://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/docs/cellxgene-demo-2.gif" width="200" height="200" hspace="30"><img src="https://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/docs/cellxgene-demo-3.gif" width="200" height="200" hspace="30">
 
-- Want to install and use cellxgene? Visit the [cellxgene docs](https://chanzuckerberg.github.io/cellxgene/).
+## getting started
-- Want to see where we are going? Check out [our roadmap](ROADMAP.md).
-- Want to contribute? See our [contributors guide](#Contributing)
 
-## quick start
+You'll need **python 3.6** and **Google Chrome**. The web UI is tested on OSX and Windows using Chrome, and the python CLI is tested on OSX and Ubuntu (via WSL/Windows). It should work on other platforms, but if you run into trouble let us know (see [help](#help-and-contact) below).
 
-To install _cellxgene_ you need Python 3.6+. We recommend [installing _cellxgene_ into a conda or virtual environment.](https://chanzuckerberg.github.io/cellxgene/faq.html#how-do-i-create-a-python-36-environment-for-cellxgene)
+To install run
 
-Install the package.
+```
-``` bash
 pip install cellxgene
 ```
 
-Download an example [anndata](https://anndata.readthedocs.io/en/latest/) file
+To start exploring a dataset call
 
-``` bash
+```
-curl -o pbmc3k.h5ad https://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/example-dataset/pbmc3k.h5ad
+cellxgene launch dataset.h5ad --open
 ```
 
-Launch _cellxgene_
+If you want an example dataset download [this file](https://github.com/chanzuckerberg/cellxgene/raw/master/example-dataset/pbmc3k.h5ad) and then call
-``` bash
+
+```
 cellxgene launch pbmc3k.h5ad --open
 ```
+You should see your web browser open with the following
 
-To learn more about what you can do with _cellxgene_, see the [Getting Started](https://chanzuckerberg.github.io/cellxgene/getting-stared/) guide.
+<img width="450" src="https://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/docs/cellxgene-opening-screenshot.png" pad="50px">
 
-## get in touch
+**Note**: automatic opening of the browser with the `--open` flag only works on OS X, on other platforms you'll need to directly point to the provided link in your browser.
 
-Have questions, suggestions, or comments? You can come hang out with us by joining the [CZI Science Slack](https://join-cellxgene-users.herokuapp.com/) and posting in the `#cellxgene-users` channel. As mentioned above, please submit any feature requests or bugs as [Github issues](https://github.com/chanzuckerberg/cellxgene/issues). We'd love to hear from you!
+There are several options available, such as:
 
-## where we are going
+- `--layout` to specify the layout as  `tsne` or `umap`
+- `--title` to show a title on the explorer
+- `--open` to automatically open the web browser after launching (OS X only)
 
-Our goal is to enable teams of computational and experimental
+To see all options call
-biologists to collaboratively gain insight into their single-cell RNA-seq data.
 
-There are 4 key features we plan to implement in the near term.
+```
+cellxgene launch --help
+```
 
-- Click install and launch
+There is an additional subcommand called `cellxgene prepare` that takes an existing dataset in one of several formats and applies minimal preprocessing and reformatting so that `launch` can use it (see [the next section](##data-formatting) for more info on `prepare`).
-- Manual annotation workflows
-- Toggle embeddings
-- Gene information
 
-For more detail on these features and where we are going, see [our roadmap](ROADMAP.md).
+## data formatting
 
-## contributing
+### assumptions
 
-We warmly welcome contributions from the community! Please submit any bug reports and feature requests through [Github issues](https://github.com/chanzuckerberg/cellxgene/issues). Please submit any direct contributions by forking the repository, creating a branch, and submitting a Pull Request. It'd be great for PRs to include test cases and documentation updates where relevant, though we know the core test suite is itself still a work in progress. And all code contributions and dependencies must be compatible with the project's open-source license (MIT). If you have any questions about this stuff, just ask!
+The `launch` command assumes that the data is stored in the `.h5ad` format from the [`anndata`](https://anndata.readthedocs.io/en/latest/index.html) library. It also assumes that certain computations have already been performed. Briefly, the `.h5ad` format wraps a two-dimensional `ndarray` and stores additional metadata as "annotations" for either observations (referred to as `obs` and `obsm`) or variables (`var` and `varm`). `cellxgene launch` makes the following assumptions about your data (we recommend loading and inspecting your data using `scanpy` to validate these assumptions)
 
-### developer guide
+- an `obs` field has a unique identifier for every cell (you can specify which field to use with the `--obs-names` option, by default it will use the value of `data.obs_names`)
+- a `var` field has a unique identifier for every gene (you can specify which field to use with the `--var-names` option, by default it will use the value of `data.var_names`)
+- an `obsm` field contains the two-dimensional coordinates for the layout that you want to render (e.g. `X_tsne` for the `tsne` layout or `X_umap` for the `umap` layout)
+- any additional  `obs` fields will be rendered as per-cell continuous or categorical metadata by the app (e.g. `louvain` cluster assignments) 
+
+### prepare
+
+The `prepare` command is included to help you format your data. It uses `scanpy` under the hood. This is especially useful if you are starting with raw unanalyzed data and are unfamiliar with `scanpy`. 
+
+To prepare from an existing `.h5ad` file use
+
+```
+cellxgene prepare dataset.h5ad --output=dataset-processed.h5ad
+```
+
+This will load the input data, perform PCA and nearest neighbor calculations, compute `umap` and `tsne` layouts and `louvain` cluster assignments, and save the results in a new file called `dataset-processed.h5ad` that can be loaded using `cellxgene launch`. Data can be loaded from several formats, including `.h5ad` `.loom` and a `10-Genomics-formatted` `mtx` directory. Several options are available, including running one of the preprocessing `recipes` included with `scanpy`, which include steps like cell filtering and gene selection. 
+
+Depending on the options chosen, `prepare` can take a long time to run (a few minutes for datasets with 10-100k cells, up to an hour or more for datasets with >100k cells). If you want `prepare` to run faster we recommend using the `sparse` option and only computing the layout for `umap`, using a call like this
+
+```
+cellxgene prepare dataset.h5ad --output=dataset-processed.h5ad --layout=umap --sparse
+```
+
+To see all options call
+
+```
+cellxgene prepare --help
+```
+
+**Note**: `cellxgene prepare` will only perform `louvain` clustering if you have the `python-igraph` and `louvain` packages installed. To make sure they are installed alongside `cellxgene` use
+
+```
+pip install cellxgene[louvain]
+```
+
+## conda and virtual environments
+
+If you use conda and want to create a conda environment for `cellxgene` you can use the following commands
+
+```
+conda create --yes -n cellxgene python=3.6
+conda activate cellxgene
+pip install cellxgene
+```
+
+Or you can create a virtual environment by using
+
+```
+ENV_NAME=cellxgene
+python3 -m venv ${ENV_NAME}
+source ${ENV_NAME}/bin/activate
+pip install cellxgene
+```
+
+## FAQ
+
+> Someone sent me a directory of `10X-Genomics` data with a `mtx` file and I've never used `scanpy`, can I use `cellxgene`?
+
+Yep! This should only take a couple steps. We'll assume your data is in a folder called `data/` and you've successfully installed `cellxgene` with the `louvain` packages as described above. Just run
+
+```
+cellxgene prepare data/ --output=data-processed.h5ad --layout=umap
+```
+
+Depending on the size of the dataset, this may take some time. Once it's done, call
+
+```
+cellxgene launch data-processed.h5ad --layout=umap --open
+```
+
+And your web browser should open with an interactive view of your data.
+
+> In my `prepare` command I received the following error `Warning: louvain module is not installed, no clusters will be calculated. To fix this please install cellxgene with the optional feature louvain enabled`
+
+Louvain clustering requires additional dependencies that are somewhat complex, so we don't include them by default. For now, you need to specify that you want these packages by using 
+
+```
+pip install cellxgene[louvain]
+```
+
+> I ran `prepare` and I'm getting results that look unexpected
+
+You might want to try running one of the preprocessing recipes included with `scanpy` (read more about them [here](https://scanpy.readthedocs.io/en/latest/api/index.html#recipes)). You can specify this with the `--recipe` option, such as
+
+```
+cellxgene prepare data/ --output=data-processed.h5ad --recipe=zheng17
+```
+
+It should be easy to run `prepare` then call `cellxgene launch` a few times with different settings to explore different behaviors. We may explore adding other preprocessing options in the future.
+
+> I have extra metadata that I want to add to my dataset
+
+Currently this is not supported directly, but you should be able to do this manually using `scanpy`. For example, this [notebook](https://github.com/falexwolf/fun-analyses/blob/master/tabula_muris/tabula_muris.ipynb) shows adding the contents of a `csv` file with metadata to an `anndata` object. For now, you could do this manually on your data in the same way and then save out the result before loading into `cellxgene`.
+
+> I tried to `pip install cellxgene` and got a weird error I don't understand
+
+This may happen, especially as we work out bugs in our installation process! Please create a new [Github issue](https://github.com/chanzuckerberg/cellxgene/issues), explain what you did, and include all the error messages you saw. It'd also be super helpful if you call `pip freeze` and include the full output alongside your issue.
+
+> How are you computing and sorting differential expression results?
+
+Currently we use a [Welch's *t*-test](https://en.wikipedia.org/wiki/Welch%27s_t-test) implementation including the same variance overestimation correction as used in `scanpy`. We sort the `tscore` to identify the top N genes, and then filter to remove any that fall below a cutoff log fold change value, which can help remove spurious test results. The default threshold is `0.01` and can be changed using the option `--diffexp-lfc-cutoff`. We can explore adding support for other test types in the future.
+
+> I'm following the developer instructions and get an error about "missing files and directories” when trying to build the client
+
+This is likely because you do not have node and npm installed, we recommend using [nvm](https://github.com/creationix/nvm) if you're new to using these tools.
+
+## developer guide
 
 This project has made a few key design choices
 
@@ -66,7 +171,7 @@ Depending on your background and interests, you might want to contribute to the
 
 If you are interested in working on `cellxgene` development, we recommend cloning the project from Gitub. First you'll need the following installed on your machine
 
-- python 3.6+
+- python 3.6
 - node and npm (we recommend using [nvm](https://github.com/creationix/nvm) if this is your first time with node)
 
 Then clone the project
@@ -75,10 +180,10 @@ Then clone the project
 git clone https://github.com/chanzuckerberg/cellxgene.git
 ```
 
-Build the client web assets by calling `make` from inside the `cellxgene` folder
+Build the client web assets by calling this from inside the `cellxgene` folder
 
 ```
-make
+./bin/build-client
 ```
 
 Install all requirements (we recommend doing this inside a virtual environment)
@@ -89,25 +194,34 @@ pip install -e .
 
 You can start the app while developing either by calling `cellxgene` or by calling `python -m server`. We recommend using the `--debug` flag to see more output, which you can include when reporting bugs.
 
-If you have any questions about developing or contributing, come hang out with us by joining the [CZI Science Slack](https://join-cellxgene-users.herokuapp.com/) and posting in the `#cellxgene-dev` channel.
+If you have any questions about developing or contributing, come hang out with us by joining the [CZI Science Slack](https://cziscience.slack.com/messages/CCTA8DF1T) and posting in the `#cellxgene-dev` channel.
 
-## inspiration
+## development roadmap
+
+`cellxgene` is still very much in development, and we've love to include the community as we plan new features to work on. We are thinking about working on the following features over the next 3-12 months. If you are interested in updates, want to give feedback, want to contribute, or have ideas about other features we should work on, please [contact us](#help-and-contact) 
+
+- **Visualizaling spatial metadata** Image-based transcriptomics methods also generate large cell by gene matrices, alongside rich metadata about spatial location; we would like to render this information in `cellxgene`
+- **Visualizing trajectories** Trajectory analyses infer progression along some ordering or pseudotime; we would like `cellxgene ` to render the results of these analyses when they have been performed
+- **Deploy to web**  Many projects release public data browser websites  alongside their publicatons; we would like to make it easy for anyone to deploy `cellxgene` to a custom URL with their own dataset that they own and operate
+- **HCA Integration** The [Human Cell Atlas](https://humancellatlas.org) is generating a large corpus of single-cell expression data and will make it available through the Data Coordination Platform; we would like `cellxgene` to be one of several different portals for browsing these data
+
+## contributing
+
+We warmly welcome contributions from the community! Please submit any bug reports and feature requests through [Github issues](https://github.com/chanzuckerberg/cellxgene/issues). Please submit any direct contributions by forking the repository, creating a branch, and submitting a Pull Request. It'd be great for PRs to include test cases and documentation updates where relevant, though we know the core test suite is itself still a work in progress. And all code contributions and dependencies must be compatible with the project's open-source license (MIT). If you have any questions about this stuff, just ask!
+
+## inspiration and collaboration
 
 We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browswer](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [Gene Pattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data.
 
 We were inspired by Mike Bostock and the [crossfilter](https://github.com/crossfilter) team for the design of our filtering implementation.
 
-We have been working closely with the [`scanpy`](https://github.com/theislab/scanpy) team to integrate with their awesome analysis tools. Special thanks to Alex Wolf, Fabian Theis, and the rest of the team for their help during development and for providing an example dataset.
+We have been working closely with the [`scanpy`](https://github.com/theislab/scanpy) team to integrate with their awesome analysis tools. Special thanks to Alex Wolf, Fabian Theis, and the rest of the team for their help during development and for providing an example dataset. 
 
 We are eager to explore integrations with other computational backends such as [`Seurat`](https://github.com/satijalab/seurat) or [`Bioconductor`](https://github.com/Bioconductor)
 
-## core team
+## help and contact
 
-- Colin Megill, frontend & product design
+Have questions, suggestions, or comments? You can come hang out with us by joining the [CZI Science Slack](https://cziscience.slack.com/messages/CCTA8DF1T) and posting in the `#cellxgene-users` channel. As mentioned above, please submit any feature requests or bugs as [Github issues](https://github.com/chanzuckerberg/cellxgene/issues). We'd love to hear from you!
-- Charlotte Weaver, software engineer
-- Bruce Martin, software engineer
-- Sidney Bell, computational biologist
-- Justin Kiggins, product manager
 
 ## reuse
 

ROADMAP.md

@@ -1,54 +0,0 @@
-# cellxgene roadmap
-
-We are very exited for _cellxgene_ to become a valuable tool in collaborations
-between computational biologists and experimental biologists working on
-single-cell transcriptomics data. _cellxgene_ is in active development, and we
-would love to include the community as we plan new features to work on. If you
-have questions of feedback about this roadmap, please submit an issue on
-GitHub.
-
-Please note: this roadmap is subject to change.
-
-*Last updated: April 11, 2019*
-
-## what we are building now
-
-In the near term, our goal is to enable teams of computational and experimental
-biologists to collaboratively explore and annotate their single-cell RNA-seq data.
-
-There are 4 key features we plan to implement in the near term.
-
-- Click install and launch
-- Manual annotation workflows
-- Toggle embeddings
-- Gene information
-
-### simple install and launch
-
-The command line interface for installing and launching cellxgene is a barrier
-for users who are not used to Python or using the command line. We plan to
-support installation and launch of cellxgene on Mac and Windows. See
-[Issue #687](https://github.com/chanzuckerberg/cellxgene/issues/687) for more details.
-
-### manual annotation workflows
-
-The exploratory visualization that cellxgene offers is critical for manual
-annotation workflows, especially in collaborative environments. We plan to
-support manually annotate cells with labels (i.e., cell type or QC flags) for
-downstream analysis. See [Issue #524](https://github.com/chanzuckerberg/cellxgene/issues/524)
-for more details.
-
-### toggle embeddings
-
-While a single dataset may have multiple embeddings calculated (tSNE, umap, in
-situ coordinates, trajectories, etc), cellxgene currently requires the user to select the
-embedding to use in the main layout at launch. We plan to support letting users
-toggle between any embedding present in a file from the cellxgene interface.
-See [Issue #594](https://github.com/chanzuckerberg/cellxgene/issues/594) for details.
-
-### gene information
-
-Differential expression returns only the names of genes, but no additional information
-about gene metadata, function, or known associations. We plan to help users learn
-more about genes they discover by exposing additional gene metadata. See
-[Issue #96](https://github.com/chanzuckerberg/cellxgene/issues/96) for details.

bin/build-client

@@ -0,0 +1,15 @@
+#!/bin/bash
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+CELLXGENE_DIR=$(dirname $DIR)
+
+cd $CELLXGENE_DIR
+
+npm install --prefix client/ client
+npm run  --prefix client build
+rm -rf server/app/web/static
+mkdir -p server/app/web/static/img
+cp client/build/index.html server/app/web/templates/
+cp -r client/build/static server/app/web/
+
+cp client/build/favicon.png server/app/web/static/img
+cp client/build/service-worker.js server/app/web/static/js/

bin/clean

@@ -0,0 +1,18 @@
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+CELLXGENE_DIR=$(dirname $DIR)
+
+echo "Uninstalling cellxgene"
+yes | pip uninstall cellxgene
+echo "removing node_modules"
+rm -rf $CELLXGENE_DIR/client/node_modules
+echo "removing client_build"
+rm -rf $CELLXGENE_DIR/client/build
+echo "removing dist"
+rm -rf $CELLXGENE_DIR/dist
+echo "removing egg-info"
+rm -rf $CELLXGENE_DIR/cellxgene.egg-info
+echo "removing static files"
+rm -f $CELLXGENE_DIR/server/app/web/templates/index.html
+rm -rf $CELLXGENE_DIR/server/app/web/static
+echo "cellxgene cleanup complete"
+

client/__tests__/e2e/config.js

@@ -1,6 +0,0 @@
-export const jest_env = process.env.JEST_ENV || "dev";
-export const appPort = process.env.JEST_CXG_PORT || 3000;
-export const appUrlBase = `http://localhost:${appPort}`;
-export const DEV = jest_env === "dev";
-export const DEBUG = jest_env === "debug";
-export const DATASET = "pbmc3k";

client/__tests__/e2e/data.js

@@ -1,122 +0,0 @@
-export const datasets = {
-  pbmc3k: {
-    title: "cellxgene: pbmc3k",
-    dataframe: {
-      nObs: "2638",
-      nVar: "1838",
-      type: "float32"
-    },
-    categorical: {
-      louvain: {
-        "B cells": "342",
-        "CD14+ Monocytes": "480",
-        "CD4 T cells": "1144",
-        "CD8 T cells": "316",
-        "Dendritic cells": "37",
-        "FCGR3A+ Monocytes": "150",
-        Megakaryocytes: "15",
-        "NK cells": "154"
-      }
-    },
-    continuous: {
-      n_genes: "int32",
-      percent_mito: "float32",
-      n_counts: "float32"
-    },
-    cellsets: {
-      lasso: [
-        {
-          "coordinates-as-percent": { x1: 0.25, y1: 0.25, x2: 0.35, y2: 0.35 },
-          count: "26"
-        }
-      ],
-      categorical: [
-        {
-          metadata: "louvain",
-          values: ["B cells", "Megakaryocytes"],
-          count: "357"
-        }
-      ],
-      continuous: [
-        {
-          metadata: "n_genes",
-          "coordinates-as-percent": { x1: 0.25, y1: 0.5, x2: 0.55, y2: 0.5 },
-          count: "1537"
-        }
-      ]
-    },
-
-    diffexp: {
-      cellset1: [
-        { kind: "categorical", metadata: "louvain", values: ["B cells"] }
-      ],
-      cellset2: [
-        {
-          kind: "categorical",
-          metadata: "louvain",
-          values: ["CD4 T cells", "NK cells"]
-        }
-      ],
-      "gene-results": [
-        "HLA-DRB1",
-        "HLA-DPB1",
-        "CD79A",
-        "HLA-DPA1",
-        "HLA-DQA1",
-        "CD79B",
-        "HLA-DQB1",
-        "MS4A1",
-        "IL32",
-        "CD37"
-      ]
-    },
-
-    genes: {
-      bulkadd: ["S100A8", "FCGR3A", "LGALS2", "GSTP1"],
-      search: "ACD"
-    },
-    subset: {
-      cellset1: [
-        {
-          kind: "categorical",
-          metadata: "louvain",
-          values: ["B cells", "Megakaryocytes"]
-        }
-      ],
-      count: "357",
-      categorical: {
-        louvain: {
-          "B cells": "342",
-          Megakaryocytes: "15"
-        }
-      },
-      lasso: {
-        "coordinates-as-percent": { x1: 0.45, y1: 0.45, x2: 0.5, y2: 0.5 },
-        count: "67"
-      }
-    },
-    scatter: {
-      genes: { x: "S100A8", y: "FCGR3A" }
-    },
-    pan: {
-      "coordinates-as-percent": { x1: 0.75, y1: 0.75, x2: 0.35, y2: 0.35 }
-    },
-    features: {
-      panzoom: {
-        lasso: {
-          "coordinates-as-percent": { x1: 0.3, y1: 0.3, x2: 0.5, y2: 0.5 },
-          count: "24"
-        }
-      }
-    },
-    clip: {
-      min: "30",
-      max: "70",
-      metadata: "n_genes",
-      gene: "S100A8",
-      "coordinates-as-percent": { x1: 0.25, y1: 0.5, x2: 0.55, y2: 0.5 },
-      count: "392",
-      "gene-cell-count": "421"
-    }
-  }
-};

client/__tests__/e2e/e2e.test.js

@@ -1,339 +0,0 @@
-/*
-Smoke test suite that will be run in Travis CI
-
-Tests included in this file are expected to be relatively stable and test core features
- */
-import puppeteer from "puppeteer";
-import { appUrlBase, DEBUG, DEV, DATASET } from "./config";
-import { puppeteerUtils, cellxgeneActions } from "./puppeteerUtils";
-import { datasets } from "./data";
-
-let browser, page, utils, cxgActions, spy;
-const browserViewport = { width: 1280, height: 960 };
-let data = datasets[DATASET];
-
-if (DEBUG) jest.setTimeout(100000);
-if (DEV) jest.setTimeout(10000);
-
-beforeAll(async () => {
-  const browserParams = DEV
-    ? { headless: false, slowMo: 5 }
-    : DEBUG
-    ? { headless: false, slowMo: 100, devtools: true }
-    : {};
-  browser = await puppeteer.launch(browserParams);
-  page = await browser.newPage();
-  await page.setViewport(browserViewport);
-  if (DEV || DEBUG) {
-    page.on("console", async msg => {
-      // If there is a console.error but an error is not thrown, this will ensure the test fails
-      if (msg.type() === "error") {
-        const errorMsgText = await Promise.all(
-          // TODO can we do this without internal properties?
-          msg.args().map(arg => arg._remoteObject.description)
-        );
-        throw new Error(`Console error: ${errorMsgText}`);
-      }
-      console.log(`PAGE LOG: ${msg.text()}`);
-    });
-  }
-  page.on("pageerror", err => {
-    throw new Error(`Console error: ${err}`);
-  });
-  utils = puppeteerUtils(page);
-  cxgActions = cellxgeneActions(page);
-});
-
-beforeEach(async () => {
-  await page.goto(appUrlBase);
-});
-
-afterAll(() => {
-  if (!DEBUG) {
-    browser.close();
-  }
-});
-
-describe("did launch", async () => {
-  test("page launched", async () => {
-    let el = await utils.getOneElementInnerHTML("[data-testid='header']");
-    expect(el).toBe(data.title);
-  });
-});
-
-describe("metadata loads", async () => {
-  test("categories and values from dataset appear", async () => {
-    for (const label in data.categorical) {
-      await utils.waitByID(`category-${label}`);
-      const categoryName = await utils.getOneElementInnerText(
-        `[data-testid="category-${label}"]`
-      );
-      expect(categoryName).toMatch(label);
-      await utils.clickOn(`category-expand-${label}`);
-      const categories = await cxgActions.getAllCategoriesAndCounts(label);
-      expect(Object.keys(categories)).toMatchObject(
-        Object.keys(data.categorical[label])
-      );
-      expect(Object.values(categories)).toMatchObject(
-        Object.values(data.categorical[label])
-      );
-    }
-  });
-
-  test("continuous data appears", async () => {
-    for (const label in data.continuous) {
-      await utils.waitByID(`histogram-${label}`);
-    }
-  });
-});
-
-describe("cell selection", async () => {
-  test("selects all cells cellset 1", async () => {
-    const cellCount = await cxgActions.cellSet(1);
-    expect(cellCount).toBe(data.dataframe.nObs);
-  });
-
-  test("selects all cells cellset 2", async () => {
-    const cellCount = await cxgActions.cellSet(2);
-    expect(cellCount).toBe(data.dataframe.nObs);
-  });
-
-  test("selects cells via lasso", async () => {
-    for (const cellset of data.cellsets.lasso) {
-      const cellset1 = await cxgActions.calcDragCoordinates(
-        "layout-graph",
-        cellset["coordinates-as-percent"]
-      );
-      await cxgActions.drag("layout-graph", cellset1.start, cellset1.end, true);
-      const cellCount = await cxgActions.cellSet(1);
-      expect(cellCount).toBe(cellset.count);
-    }
-  });
-
-  test("selects cells via categorical", async () => {
-    for (const cellset of data.cellsets.categorical) {
-      await utils.clickOn(`category-expand-${cellset.metadata}`);
-      await utils.clickOn(`category-select-${cellset.metadata}`);
-      for (const val of cellset.values) {
-        await utils.clickOn(
-          `categorical-value-select-${cellset.metadata}-${val}`
-        );
-      }
-      const cellCount = await cxgActions.cellSet(1);
-      expect(cellCount).toBe(cellset.count);
-    }
-  });
-
-  test("selects cells via continuous", async () => {
-    for (const cellset of data.cellsets.continuous) {
-      const histId = `histogram-${cellset.metadata}-plot-brush`;
-      const coords = await cxgActions.calcDragCoordinates(
-        histId,
-        cellset["coordinates-as-percent"]
-      );
-      await cxgActions.drag(histId, coords.start, coords.end);
-      const cellCount = await cxgActions.cellSet(1);
-      expect(cellCount).toBe(cellset.count);
-    }
-  });
-});
-
-describe("gene entry", async () => {
-  test("search for single gene", async () => {
-    // blueprint's  typeahead is treating typing weird, clicking & waiting first solves this
-    await utils.typeInto("gene-search", data.genes.search);
-    await page.keyboard.press("Enter");
-    await page.waitForSelector(
-      `[data-testid='histogram-${data.genes.search}']`
-    );
-  });
-
-  test("bulk add genes", async () => {
-    await cxgActions.reset();
-    const testGenes = data.genes.bulkadd;
-    await utils.clickOn("section-bulk-add");
-    await utils.typeInto("input-bulk-add", testGenes.join(","));
-    await page.keyboard.press("Enter");
-    const userGeneHist = await cxgActions.getAllHistograms(
-      "histogram-user-gene"
-    );
-    expect(userGeneHist).toEqual(expect.arrayContaining(testGenes));
-  });
-});
-
-describe("diffexp", async () => {
-  test("selects cells, saves them and performs diffexp", async () => {
-    for (const select of data.diffexp.cellset1) {
-      if (select.kind === "categorical") {
-        await cxgActions.selectCategory(select.metadata, select.values, true);
-      }
-    }
-    await cxgActions.cellSet(1);
-    for (const select of data.diffexp.cellset2) {
-      if (select.kind === "categorical") {
-        await cxgActions.selectCategory(select.metadata, select.values, true);
-      }
-    }
-    await cxgActions.cellSet(2);
-    await utils.clickOn("diffexp-button");
-    const diffExpHists = await cxgActions.getAllHistograms("histogram-diffexp");
-    expect(diffExpHists).toEqual(
-      expect.arrayContaining(data.diffexp["gene-results"])
-    );
-  });
-});
-
-describe("subset/reset", async () => {
-  test("subset - cell count matches", async () => {
-    for (const select of data.subset.cellset1) {
-      if (select.kind === "categorical") {
-        await cxgActions.selectCategory(select.metadata, select.values, true);
-      }
-    }
-    await utils.clickOn("subset-button");
-    for (const label in data.subset.categorical) {
-      const categories = await cxgActions.getAllCategoriesAndCounts(label);
-      expect(Object.keys(categories)).toMatchObject(
-        Object.keys(data.subset.categorical[label])
-      );
-      expect(Object.values(categories)).toMatchObject(
-        Object.values(data.subset.categorical[label])
-      );
-    }
-  });
-
-  test("reset after subset", async () => {
-    for (const select of data.subset.cellset1) {
-      if (select.kind === "categorical") {
-        await cxgActions.selectCategory(select.metadata, select.values, true);
-      }
-    }
-    await utils.clickOn("subset-button");
-    for (const label in data.subset.categorical) {
-      const categories = await cxgActions.getAllCategoriesAndCounts(label);
-      expect(Object.keys(categories)).toMatchObject(
-        Object.keys(data.subset.categorical[label])
-      );
-      expect(Object.values(categories)).toMatchObject(
-        Object.values(data.subset.categorical[label])
-      );
-    }
-    await cxgActions.reset();
-    for (const label in data.categorical) {
-      await utils.waitByID(`category-${label}`);
-      const categoryName = await utils.getOneElementInnerText(
-        `[data-testid="category-${label}"]`
-      );
-      expect(categoryName).toMatch(label);
-      const categories = await cxgActions.getAllCategoriesAndCounts(label);
-      expect(Object.keys(categories)).toMatchObject(
-        Object.keys(data.categorical[label])
-      );
-      expect(Object.values(categories)).toMatchObject(
-        Object.values(data.categorical[label])
-      );
-    }
-  });
-
-  test("lasso after subset", async () => {
-    for (const select of data.subset.cellset1) {
-      if (select.kind === "categorical") {
-        await cxgActions.selectCategory(select.metadata, select.values, true);
-      }
-    }
-    await utils.clickOn("subset-button");
-    const lassoSelection = await cxgActions.calcDragCoordinates(
-      "layout-graph",
-      data.subset.lasso["coordinates-as-percent"]
-    );
-    await cxgActions.drag(
-      "layout-graph",
-      lassoSelection.start,
-      lassoSelection.end,
-      true
-    );
-    const cellCount = await cxgActions.cellSet(1);
-    expect(cellCount).toBe(data.subset.lasso.count);
-  });
-});
-
-describe("scatter plot", async () => {
-  test("scatter plot appears", async () => {
-    await cxgActions.reset();
-    const testGenes = data.scatter.genes;
-    await utils.clickOn("section-bulk-add");
-    await utils.typeInto("input-bulk-add", Object.values(testGenes).join(","));
-    await page.keyboard.press("Enter");
-    await utils.clickOn(`plot-x-${data.scatter.genes.x}`);
-    await utils.clickOn(`plot-y-${data.scatter.genes.y}`);
-    await utils.waitByID("scatterplot");
-  });
-});
-
-describe("clipping", async () => {
-  test("clip continuous", async () => {
-    await cxgActions.clip(data.clip.min, data.clip.max)
-    const histId = `histogram-${data.clip.metadata}-plot-brush`;
-    const coords = await cxgActions.calcDragCoordinates(
-      histId,
-      data.clip["coordinates-as-percent"]
-    );
-    await cxgActions.drag(histId, coords.start, coords.end);
-    const cellCount = await cxgActions.cellSet(1);
-    expect(cellCount).toBe(data.clip.count);
-    
-  });
-
-  test("clip gene", async () => {
-    await utils.typeInto("gene-search", data.clip.gene);
-    await page.keyboard.press("Enter");
-    await page.waitForSelector(
-      `[data-testid='histogram-${data.clip.gene}']`
-    );
-    await cxgActions.clip(data.clip.min, data.clip.max)
-    const histId = `histogram-${data.clip.gene}-plot-brush`;
-    const coords = await cxgActions.calcDragCoordinates(
-      histId,
-      data.clip["coordinates-as-percent"]
-    );
-    await cxgActions.drag(histId, coords.start, coords.end);
-    const cellCount = await cxgActions.cellSet(1);
-    expect(cellCount).toBe(data.clip["gene-cell-count"]);
-  });
-});
-
-// interact with UI elements just that they do not break
-describe("ui elements don't error", async () => {
-  test("color by", async () => {
-    for (const label in data.categorical) {
-      await utils.clickOn(`colorby-${label}`);
-    }
-    for (const label in data.continuous) {
-      await utils.clickOn(`colorby-${label}`);
-    }
-  });
-
-  test("color by for gene", async () => {
-    await utils.typeInto("gene-search", data.genes.search);
-    await page.keyboard.press("Enter");
-    await page.waitForSelector(
-      `[data-testid='histogram-${data.genes.search}']`
-    );
-    await utils.clickOn(`colorby-${data.genes.search}`);
-  });
-
-  test("pan and zoom", async () => {
-    await utils.clickOn("mode-pan-zoom");
-    const panCoords = await cxgActions.calcDragCoordinates(
-      "layout-graph",
-      data.pan["coordinates-as-percent"]
-    );
-    await cxgActions.drag(
-      "layout-graph",
-      panCoords.start,
-      panCoords.end,
-      false
-    );
-    await page.evaluate(`window.scrollBy(0, 1000);`);
-  });
-});

client/__tests__/e2e/e2eJestConfig.json

@@ -1,10 +0,0 @@
-{
-  "preset": "jest-puppeteer",
-  "testMatch": [
-    "**/__tests__/**/?(*.)(spec|test).js?(x)"
-  ],
-  "testURL": "http://localhost/",
-  "setupFiles": [
-    "../setupMissingGlobals.js"
-  ]
-}

client/__tests__/e2e/feature.test.js

@@ -1,118 +0,0 @@
-/*
-NOT run in Travis CI
-
-UX tests using puppeteer to be run locally.
-
-To run locally, ensure you are running the client is running on port 3000.
-Then run jest --verbose false --config __tests__/e2e/e2eJestConfig.json feature.
- */
-
-import puppeteer from "puppeteer";
-import { appUrlBase, DEBUG, DEV, DATASET } from "./config";
-import { puppeteerUtils, cellxgeneActions } from "./puppeteerUtils";
-import { datasets } from "./data";
-
-let browser, page, utils, cxgActions, spy;
-const browserViewport = { width: 1280, height: 960 };
-let data = datasets[DATASET].features;
-
-if (DEBUG) jest.setTimeout(100000);
-if (DEV) jest.setTimeout(10000);
-
-beforeAll(async () => {
-  const browserParams = DEV
-    ? { headless: false, slowMo: 5 }
-    : DEBUG
-    ? { headless: false, slowMo: 100, devtools: true }
-    : {};
-  browser = await puppeteer.launch(browserParams);
-  page = await browser.newPage();
-  await page.setViewport(browserViewport);
-  if (DEV || DEBUG) {
-    page.on("console", msg => console.log(`PAGE LOG: ${msg.text()}`));
-  }
-  page.on("pageerror", err => {
-    throw new Error(`Console error: ${err}`);
-  });
-  utils = puppeteerUtils(page);
-  cxgActions = cellxgeneActions(page);
-});
-
-beforeEach(async () => {
-  await page.goto(appUrlBase);
-});
-
-afterAll(() => {
-  if (!DEBUG) {
-    browser.close();
-  }
-});
-
-describe("zoom interaction", async () => {
-  // Skip this test since UI is to hide lasso path when switching modes
-  test.skip("lasso visible after switching modes to pan/zoom", async () => {
-    const lassoSelection = await cxgActions.calcDragCoordinates(
-      "layout-graph",
-      data.panzoom.lasso["coordinates-as-percent"]
-    );
-    await cxgActions.drag(
-      "layout-graph",
-      lassoSelection.start,
-      lassoSelection.end,
-      true
-    );
-    await utils.waitByID("lasso-element", { visible: true });
-    await utils.clickOn("mode-pan-zoom");
-    await utils.waitByID("lasso-element", { visible: true });
-  });
-
-  test("pan zoom mode resets lasso selection", async () => {
-    const lassoSelection = await cxgActions.calcDragCoordinates(
-      "layout-graph",
-      data.panzoom.lasso["coordinates-as-percent"]
-    );
-    await cxgActions.drag(
-      "layout-graph",
-      lassoSelection.start,
-      lassoSelection.end,
-      true
-    );
-    await utils.waitByID("lasso-element", { visible: true });
-    const initialCount = await cxgActions.cellSet(1);
-    expect(initialCount).toBe(data.panzoom.lasso.count);
-    await utils.clickOn("mode-pan-zoom");
-    await utils.clickOn("mode-lasso");
-    const modeSwitchCount = await cxgActions.cellSet(1);
-    expect(modeSwitchCount).toBe(initialCount);
-  });
-
-  test("lasso moves after pan", async () => {
-    const lassoSelection = await cxgActions.calcDragCoordinates(
-      "layout-graph",
-      data.panzoom.lasso["coordinates-as-percent"]
-    );
-    await cxgActions.drag(
-      "layout-graph",
-      lassoSelection.start,
-      lassoSelection.end,
-      true
-    );
-    await utils.waitByID("lasso-element", { visible: true });
-    const initialCount = await cxgActions.cellSet(1);
-    expect(initialCount).toBe(data.panzoom.lasso.count);
-    await utils.clickOn("mode-pan-zoom");
-    const panCoords = await cxgActions.calcDragCoordinates(
-      "layout-graph",
-      data.panzoom.lasso["coordinates-as-percent"]
-    );
-    await cxgActions.drag(
-      "layout-graph",
-      panCoords.start,
-      panCoords.end,
-      false
-    );
-    await utils.clickOn("mode-lasso");
-    const panCount = await cxgActions.cellSet(2);
-    expect(panCount).toBe(initialCount);
-  });
-});

client/__tests__/e2e/puppeteerUtils.js

@@ -1,188 +0,0 @@
-export const puppeteerUtils = puppeteerPage => ({
-  async waitByID(testid, props = {}) {
-    return await puppeteerPage.waitForSelector(
-      `[data-testid='${testid}']`,
-      props
-    );
-  },
-
-  async waitByClass(testclass, props = {}) {
-    return await puppeteerPage.waitForSelector(
-      `[data-testclass='${testclass}']`,
-      props
-    );
-  },
-
-  async typeInto(testid, text) {
-    // only works for text without special characters
-    await this.waitByID(testid);
-    const selector = `[data-testid='${testid}']`;
-    // type ahead can be annoying if you don't pause before you type
-    await puppeteerPage.click(selector);
-    await puppeteerPage.waitFor(200);
-    await puppeteerPage.type(selector, text);
-  },
-
-  async clearInputAndTypeInto(testid, text) {
-    await this.waitByID(testid);
-    const selector = `[data-testid='${testid}']`;
-    // only works for text without special characters
-    // type ahead can be annoying if you don't pause before you type
-    await puppeteerPage.click(selector);
-    await puppeteerPage.waitFor(200);
-    // select all
-
-    await puppeteerPage.click(selector, {clickCount: 3})
-    await puppeteerPage.keyboard.type("Backspace")
-    await puppeteerPage.type(selector, text);
-  },
-
-  async clickOn(testid) {
-    await this.waitByID(testid);
-    await puppeteerPage.click(`[data-testid='${testid}']`);
-    await puppeteerPage.waitFor(50);
-  },
-
-  async getOneElementInnerHTML(selector) {
-    let text = await puppeteerPage.$eval(selector, el => el.innerHTML);
-    return text;
-  },
-
-  async getOneElementInnerText(selector) {
-    let text = await puppeteerPage.$eval(selector, el => el.innerText);
-    return text;
-  }
-});
-
-export const cellxgeneActions = puppeteerPage => ({
-  async drag(testid, start, end, lasso = false) {
-    const layout = await puppeteerUtils(puppeteerPage).waitByID(testid);
-    const elBox = await layout.boxModel();
-    const x1 = elBox.content[0].x + start.x;
-    const x2 = elBox.content[0].x + end.x;
-    const y1 = elBox.content[0].y + start.y;
-    const y2 = elBox.content[0].y + end.y;
-    await puppeteerPage.mouse.move(x1, y1);
-    await puppeteerPage.mouse.down();
-    if (lasso) {
-      await puppeteerPage.mouse.move(x2, y1);
-      await puppeteerPage.mouse.move(x2, y2);
-      await puppeteerPage.mouse.move(x1, y2);
-      await puppeteerPage.mouse.move(x1, y1);
-    } else {
-      await puppeteerPage.mouse.move(x2, y2);
-    }
-    await puppeteerPage.mouse.up();
-  },
-
-  async getAllHistograms(testclass) {
-    await puppeteerUtils(puppeteerPage).waitByClass(testclass);
-    const histograms = await puppeteerPage.$$eval(
-      `[data-testclass=${testclass}]`,
-      els => {
-        return els.map(el => {
-          return el.dataset.testid.substring(
-            "histogram_".length,
-            el.dataset.testid.length
-          );
-        });
-      }
-    );
-    return histograms;
-  },
-
-  async getAllCategoriesAndCounts(category) {
-    await puppeteerUtils(puppeteerPage).waitByClass("categorical-row");
-    const categories = await puppeteerPage.$$eval(
-      `[data-testid="category-${category}"] [data-testclass='categorical-row']`,
-      els => {
-        let result = {};
-        els.forEach(el => {
-          const cat = el.querySelector("[data-testclass='categorical-value']")
-            .innerText;
-          const count = el.querySelector(
-            "[data-testclass='categorical-value-count']"
-          ).innerText;
-          result[cat] = count;
-        });
-        return result;
-      }
-    );
-    return categories;
-  },
-
-  async cellSet(num) {
-    await puppeteerUtils(puppeteerPage).clickOn(`cellset-button-${num}`);
-    return await puppeteerUtils(puppeteerPage).getOneElementInnerText(
-      `[data-testid='cellset-count-${num}']`
-    );
-  },
-
-  async resetCategory(category) {
-    const checkboxId = `category-select-${category}`;
-    await puppeteerUtils(puppeteerPage).waitByID(checkboxId);
-    const checkedPseudoclass = await puppeteerPage.$eval(
-      `[data-testid='${checkboxId}']`,
-      el => {
-        return el.matches(":checked");
-      }
-    );
-    if (!checkedPseudoclass) {
-      await puppeteerUtils(puppeteerPage).clickOn(checkboxId);
-    }
-    try {
-      const categoryRow = await puppeteerUtils(puppeteerPage).waitByID(
-        `category-expand-${category}`
-      );
-      const isExpanded = await categoryRow.$(
-        "[data-testclass='category-expand-is-expanded']"
-      );
-      if (isExpanded) {
-        await puppeteerUtils(puppeteerPage).clickOn(
-          `category-expand-${category}`
-        );
-      }
-    } catch {}
-  },
-
-  async calcDragCoordinates(testid, coordinateAsPercent) {
-    const el = await puppeteerUtils(puppeteerPage).waitByID(testid);
-    const size = await el.boxModel();
-    const coords = {
-      start: {
-        x: Math.floor(size.width * coordinateAsPercent.x1),
-        y: Math.floor(size.height * coordinateAsPercent.y1)
-      },
-      end: {
-        x: Math.floor(size.width * coordinateAsPercent.x2),
-        y: Math.floor(size.height * coordinateAsPercent.y2)
-      }
-    };
-    return coords;
-  },
-
-  async selectCategory(category, values, reset = true) {
-    if (reset) await this.resetCategory(category);
-    await puppeteerUtils(puppeteerPage).clickOn(`category-expand-${category}`);
-    await puppeteerUtils(puppeteerPage).clickOn(`category-select-${category}`);
-    for (const val of values) {
-      await puppeteerUtils(puppeteerPage).clickOn(
-        `categorical-value-select-${category}-${val}`
-      );
-    }
-  },
-
-  async reset() {
-    await puppeteerUtils(puppeteerPage).clickOn("reset");
-    // loading state never actually happens, reset is too fast
-    await page.waitFor(200);
-  },
-
-  async clip(min = 0, max = 100) { 
-    await puppeteerUtils(puppeteerPage).clickOn("visualization-settings");
-    await puppeteerUtils(puppeteerPage).clearInputAndTypeInto("clip-min-input", min);
-    await puppeteerUtils(puppeteerPage).clearInputAndTypeInto("clip-max-input", max);
-    await puppeteerUtils(puppeteerPage).clickOn("clip-commit");
-  }
-
-});

client/__tests__/reducers/cascade.test.js

@@ -1,48 +0,0 @@
-import cascadeReducers from "../../src/reducers/cascade";
-
-describe("create", () => {
-  test("from Array", () => {
-    expect(cascadeReducers([["foo", () => 0]])).toBeInstanceOf(Function);
-  });
-
-  test("from Map", () => {
-    expect(cascadeReducers(new Map([["foo", () => 0]]))).toBeInstanceOf(
-      Function
-    );
-  });
-});
-
-describe("cascade", () => {
-  test("expected arguments provided & cascade ordering", () => {
-    const topLevelState = {};
-    const topLevelAction = { type: "test" };
-
-    const reducer = cascadeReducers([
-      [
-        "foo",
-        (currentState, action, nextSharedState, prevSharedState) => {
-          expect(currentState).toBeUndefined();
-          expect(action).toEqual(topLevelAction);
-          expect(nextSharedState).toStrictEqual({});
-          expect(prevSharedState).toBe(topLevelState);
-          return 0;
-        }
-      ],
-      [
-        "bar",
-        (currentState, action, nextSharedState, prevSharedState) => {
-          expect(currentState).toBeUndefined();
-          expect(action).toEqual(topLevelAction);
-          expect(nextSharedState).toStrictEqual({ foo: 0 });
-          expect(prevSharedState).toBe(topLevelState);
-          return 99;
-        }
-      ]
-    ]);
-
-    const nextState = reducer(topLevelState, topLevelAction);
-    expect(nextState).toStrictEqual({ foo: 0, bar: 99 });
-    expect(topLevelState).toStrictEqual({});
-    expect(topLevelAction).toStrictEqual({ type: "test" });
-  });
-});

client/__tests__/reducers/undoable.test.js

@@ -1,87 +0,0 @@
-import undoable from "../../src/reducers/undoable";
-
-describe("create", () => {
-  test("no keys", () => {
-    expect(() => undoable(() => {})).toThrow();
-    expect(() => undoable(() => {}, null)).toThrow();
-    expect(() => undoable(() => {}, [])).toThrow();
-    expect(() => undoable(() => {}, [], {})).toThrow();
-  });
-
-  test("simple", () => {
-    expect(undoable(() => {}, ["foo"])).toBeInstanceOf(Function);
-    expect(undoable(() => {}, ["foo"], {})).toBeInstanceOf(Function);
-  });
-
-  test("handles undefined initial state", () => {
-    expect(
-      undoable(() => {}, ["a"])(undefined, { type: "test" })
-    ).toMatchObject({});
-  });
-});
-
-describe("undo", () => {
-  test("expected state modifications", () => {
-    const initialState = { a: 0, b: 1000 };
-    const reducer = state => {
-      return { a: state.a + 1, b: state.b + 1 };
-    };
-    const undoableReducer = undoable(reducer, ["a"]);
-
-    const s1 = undoableReducer(initialState, { type: "test" });
-    expect(s1).toMatchObject({ a: 1, b: 1001 });
-
-    // test that only specified keys are undone
-    const s2 = undoableReducer(s1, { type: "@@undoable/undo" });
-    expect(s2).toMatchObject({ a: 0, b: 1001 });
-
-    // test backstop when no more history
-    const s3 = undoableReducer(s2, { type: "@@undoable/undo" });
-    expect(s3).toMatchObject({ a: 0, b: 1001 });
-  });
-});
-
-describe("redo", () => {
-  const initialState = { a: 0, b: 1000 };
-  const reducer = state => {
-    return { a: state.a + 1, b: state.b + 1 };
-  };
-  let UR;
-
-  beforeEach(() => {
-    UR = undoable(reducer, ["a"]);
-  });
-
-  test("expected state modifications", () => {
-    const s1 = UR(initialState, { type: "test" });
-    expect(s1).toMatchObject({ a: 1, b: 1001 });
-
-    // verify undo->redo reverts state.
-    const s2 = UR(UR(s1, { type: "@@undoable/undo" }), {
-      type: "@@undoable/redo"
-    });
-    expect(s2).toMatchObject({ a: 1, b: 1001 });
-
-    // verify backstop when no redo future
-    const s3 = UR(s2, { type: "@@undoable/redo" });
-    expect(s3).toMatchObject({ a: 1, b: 1001 });
-  });
-
-  test("history cleared", () => {
-    // verify future cleared upon a normal state transition
-    const s1 = UR(initialState, { type: "test" });
-    expect(s1).toMatchObject({ a: 1, b: 1001 });
-    const s2 = UR(s1, { type: "@@undoable/undo" });
-    expect(s2).toMatchObject({ a: 0, b: 1001 });
-    const s3 = UR(s2, { type: "test" });
-    expect(s3).toMatchObject({ a: 1, b: 1002 });
-    const s4 = UR(s3, { type: "@@undoable/redo" });
-    expect(s4).toMatchObject({ a: 1, b: 1002 });
-  });
-});
-
-/*
-TODO:
-- historyLimit is enforced
-- action filters
-*/

client/__tests__/setupMissingGlobals.js

@@ -1,9 +0,0 @@
-/*
-Define globals which are present in the client, but not in node (and therefore not in
-the jest test environment).
-*/
-
-import { TextDecoder, TextEncoder } from "util";
-
-global.TextDecoder = TextDecoder;
-global.TextEncoder = TextEncoder;

client/__tests__/util/dataframe/dataframe.test.js

@@ -1,652 +0,0 @@
-import * as Dataframe from "../../../src/util/dataframe";
-
-describe("dataframe constructor", () => {
-  test("empty dataframe", () => {
-    const df = new Dataframe.Dataframe([0, 0], []);
-    expect(df).toBeDefined();
-    expect(df.dims).toEqual([0, 0]);
-    expect(df).toHaveLength(0);
-    expect(df.icol(0)).not.toBeDefined();
-  });
-
-  test("create with default indices", () => {
-    const df = new Dataframe.Dataframe(
-      [3, 2],
-      [new Int32Array(3).fill(0), new Int32Array(3).fill(1)]
-    );
-
-    expect(df).toBeDefined();
-    expect(df.dims).toEqual([3, 2]);
-    expect(df.rowIndex).toBeInstanceOf(Dataframe.IdentityInt32Index);
-    expect(df.colIndex).toBeInstanceOf(Dataframe.IdentityInt32Index);
-    expect(df.at(0, 0)).toEqual(0);
-    expect(df.at(2, 1)).toEqual(1);
-    expect(df.iat(0, 0)).toEqual(0);
-    expect(df.iat(2, 1)).toEqual(1);
-  });
-
-  test("create with labelled indices", () => {
-    const df = new Dataframe.Dataframe(
-      [3, 2],
-      [new Int32Array([0, 1, 2]), new Int32Array([3, 4, 5])],
-      new Dataframe.DenseInt32Index([2, 1, 0]),
-      new Dataframe.KeyIndex(["A", "B"])
-    );
-
-    expect(df).toBeDefined();
-    expect(df.dims).toEqual([3, 2]);
-
-    expect(df.rowIndex).toBeInstanceOf(Dataframe.DenseInt32Index);
-    expect(df.colIndex).toBeInstanceOf(Dataframe.KeyIndex);
-    expect(df.rowIndex.keys()).toEqual(new Int32Array([2, 1, 0]));
-    expect(df.colIndex.keys()).toEqual(["A", "B"]);
-
-    expect(df.at(0, "A")).toEqual(2);
-    expect(df.at(2, "B")).toEqual(3);
-    expect(df.iat(0, 0)).toEqual(0);
-    expect(df.iat(2, 1)).toEqual(5);
-  });
-});
-
-describe("simple data access", () => {
-  const df = new Dataframe.Dataframe(
-    [4, 2],
-    [
-      new Float64Array([0.0, Number.NaN, Number.POSITIVE_INFINITY, 3.14159]),
-      ["red", "blue", "green", "nan"]
-    ],
-    new Dataframe.DenseInt32Index([3, 2, 1, 0]),
-    new Dataframe.KeyIndex(["numbers", "colors"])
-  );
-
-  test("iat", () => {
-    expect(df).toBeDefined();
-
-    // present
-    expect(df.iat(0, 0)).toEqual(0.0);
-    expect(df.iat(0, 1)).toEqual("red");
-    expect(df.iat(1, 0)).toEqual(Number.NaN);
-    expect(df.iat(1, 1)).toEqual("blue");
-    expect(df.iat(2, 0)).toEqual(Number.POSITIVE_INFINITY);
-    expect(df.iat(2, 1)).toEqual("green");
-    expect(df.iat(3, 0)).toEqual(3.14159);
-    expect(df.iat(3, 1)).toEqual("nan");
-
-    // labels out of range have no defined behavior
-  });
-
-  test("at", () => {
-    expect(df).toBeDefined();
-
-    // present
-    expect(df.at(3, "numbers")).toEqual(0.0);
-    expect(df.at(3, "colors")).toEqual("red");
-    expect(df.at(2, "numbers")).toEqual(Number.NaN);
-    expect(df.at(2, "colors")).toEqual("blue");
-    expect(df.at(1, "numbers")).toEqual(Number.POSITIVE_INFINITY);
-    expect(df.at(1, "colors")).toEqual("green");
-    expect(df.at(0, "numbers")).toEqual(3.14159);
-    expect(df.at(0, "colors")).toEqual("nan");
-
-    // labels out of range have no defined behavior
-  });
-
-  test("ihas", () => {
-    expect(df).toBeDefined();
-
-    // present
-    expect(df.ihas(0, 0)).toBeTruthy();
-    expect(df.ihas(1, 1)).toBeTruthy();
-    expect(df.ihas(3, 1)).toBeTruthy();
-
-    // not present
-    expect(df.ihas(-1, -1)).toBeFalsy();
-    expect(df.ihas(0, 99)).toBeFalsy();
-    expect(df.ihas(99, 0)).toBeFalsy();
-    expect(df.ihas(99, 99)).toBeFalsy();
-    expect(df.ihas(-1, 0)).toBeFalsy();
-    expect(df.ihas(0, -1)).toBeFalsy();
-  });
-
-  test("has", () => {
-    expect(df).toBeDefined();
-
-    // present
-    expect(df.has(3, "numbers")).toBeTruthy();
-    expect(df.has(0, "numbers")).toBeTruthy();
-    expect(df.has(3, "colors")).toBeTruthy();
-    expect(df.has(0, "colors")).toBeTruthy();
-
-    // not present
-    expect(df.has(3, "foo")).toBeFalsy();
-    expect(df.has(-1, "numbers")).toBeFalsy();
-    expect(df.has(-1, -1)).toBeFalsy();
-    expect(df.has(null, null)).toBeFalsy();
-    expect(df.has(0, "foo")).toBeFalsy();
-    expect(df.has(99, "numbers")).toBeFalsy();
-    expect(df.has(99, "foo")).toBeFalsy();
-  });
-});
-
-describe("dataframe subsetting", () => {
-  describe("subset", () => {
-    const sourceDf = new Dataframe.Dataframe(
-      [3, 4],
-      [
-        new Int32Array([0, 1, 2]),
-        ["A", "B", "C"],
-        new Float32Array([4.4, 5.5, 6.6]),
-        ["red", "green", "blue"]
-      ],
-      null,
-      new Dataframe.KeyIndex(["int32", "string", "float32", "colors"])
-    );
-
-    test("all rows, one column", () => {
-      const dfA = sourceDf.subset(null, ["colors"]);
-      expect(dfA).toBeDefined();
-      expect(dfA.dims).toEqual([3, 1]);
-      expect(dfA.iat(0, 0)).toEqual("red");
-      expect(dfA.at(2, "colors")).toEqual("blue");
-      expect(dfA.col("colors").asArray()).toEqual(["red", "green", "blue"]);
-      expect(dfA.icol(0).asArray()).toEqual(["red", "green", "blue"]);
-      expect(dfA.col("colors").asArray()).toEqual(
-        sourceDf.col("colors").asArray()
-      );
-      expect(dfA.rowIndex.keys()).toEqual(sourceDf.rowIndex.keys());
-      expect(dfA.colIndex.keys()).toEqual(["colors"]);
-    });
-
-    test("all rows, two columns", () => {
-      const dfB = sourceDf.subset(null, ["colors", "float32"]);
-      expect(dfB).toBeDefined();
-      expect(dfB.dims).toEqual([3, 2]);
-      expect(dfB.iat(0, 0)).toBeCloseTo(4.4);
-      expect(dfB.iat(0, 1)).toEqual("red");
-      expect(dfB.at(2, "colors")).toEqual("blue");
-      expect(dfB.at(2, "float32")).toBeCloseTo(6.6);
-      expect(dfB.col("colors").asArray()).toEqual(["red", "green", "blue"]);
-      expect(dfB.col("float32").asArray()).toEqual(
-        new Float32Array([4.4, 5.5, 6.6])
-      );
-      expect(dfB.icol(0).asArray()).toEqual(dfB.col("float32").asArray());
-      expect(dfB.icol(1).asArray()).toEqual(dfB.col("colors").asArray());
-      expect(dfB.col("colors").asArray()).toEqual(
-        sourceDf.col("colors").asArray()
-      );
-      expect(dfB.col("float32").asArray()).toEqual(
-        sourceDf.col("float32").asArray()
-      );
-      expect(dfB.rowIndex.keys()).toEqual(sourceDf.rowIndex.keys());
-      expect(dfB.colIndex.keys()).toEqual(["float32", "colors"]);
-    });
-
-    test("one row, all columns", () => {
-      const dfC = sourceDf.subset([1], null);
-      expect(dfC).toBeDefined();
-      expect(dfC.dims).toEqual([1, 4]);
-      expect(dfC.iat(0, 0)).toEqual(1);
-      expect(dfC.iat(0, 1)).toEqual("B");
-      expect(dfC.iat(0, 2)).toBeCloseTo(5.5);
-      expect(dfC.iat(0, 3)).toEqual("green");
-      expect(dfC.rowIndex.keys()).toEqual(new Int32Array([1]));
-      expect(dfC.colIndex.keys()).toEqual(sourceDf.colIndex.keys());
-    });
-
-    test("two rows, all columns", () => {
-      const dfD = sourceDf.subset([0, 2], null);
-      expect(dfD).toBeDefined();
-      expect(dfD.dims).toEqual([2, 4]);
-      expect(dfD.icol(0).asArray()).toEqual(new Int32Array([0, 2]));
-      expect(dfD.icol(1).asArray()).toEqual(["A", "C"]);
-      expect(dfD.icol(2).asArray()).toEqual(new Float32Array([4.4, 6.6]));
-      expect(dfD.icol(3).asArray()).toEqual(["red", "blue"]);
-      expect(dfD.rowIndex.keys()).toEqual(new Int32Array([0, 2]));
-      expect(dfD.colIndex.keys()).toEqual(sourceDf.colIndex.keys());
-    });
-
-    test("all rows, all columns", () => {
-      const dfE = sourceDf.subset(null, null);
-      expect(dfE).toBeDefined();
-      expect(dfE.dims).toEqual([3, 4]);
-      expect(dfE.icol(0).asArray()).toEqual(sourceDf.icol(0).asArray());
-      expect(dfE.icol(1).asArray()).toEqual(sourceDf.icol(1).asArray());
-      expect(dfE.icol(2).asArray()).toEqual(sourceDf.icol(2).asArray());
-      expect(dfE.icol(3).asArray()).toEqual(sourceDf.icol(3).asArray());
-      expect(dfE.rowIndex.keys()).toEqual(sourceDf.rowIndex.keys());
-      expect(dfE.colIndex.keys()).toEqual(sourceDf.colIndex.keys());
-    });
-
-    test("two rows, two colums", () => {
-      const dfF = sourceDf.subset([0, 2], ["int32", "float32"]);
-      expect(dfF).toBeDefined();
-      expect(dfF.dims).toEqual([2, 2]);
-      expect(dfF.icol(0).asArray()).toEqual(new Int32Array([0, 2]));
-      expect(dfF.icol(1).asArray()).toEqual(new Float32Array([4.4, 6.6]));
-      expect(dfF.rowIndex.keys()).toEqual(new Int32Array([0, 2]));
-      expect(dfF.colIndex.keys()).toEqual(["int32", "float32"]);
-    });
-
-    test("withRowIndex", () => {
-      const df = sourceDf.subset(
-        null,
-        ["int32", "float32"],
-        new Dataframe.DenseInt32Index([3, 2, 1])
-      );
-      expect(df.colIndex).toBeInstanceOf(Dataframe.KeyIndex);
-      expect(df.rowIndex).toBeInstanceOf(Dataframe.DenseInt32Index);
-      expect(df.at(3, "int32")).toEqual(df.iat(0, 0));
-    });
-
-    test("withRowIndex error checks", () => {
-      expect(() =>
-        sourceDf.subset(null, ["red"], new Dataframe.IdentityInt32Index(1))
-      ).toThrow(RangeError);
-      expect(() =>
-        sourceDf.subset(null, ["red"], new Dataframe.DenseInt32Index([0, 1]))
-      ).toThrow(RangeError);
-      expect(() =>
-        sourceDf.subset(null, ["red"], new Dataframe.KeyIndex([0, 1, 2, 3]))
-      ).toThrow(RangeError);
-    });
-  });
-
-  test("isubsetMask", () => {
-    const sourceDf = new Dataframe.Dataframe(
-      [3, 4],
-      [
-        new Int32Array([0, 1, 2]),
-        ["A", "B", "C"],
-        new Float32Array([4.4, 5.5, 6.6]),
-        ["red", "green", "blue"]
-      ],
-      new Dataframe.DenseInt32Index([2, 4, 6]),
-      new Dataframe.KeyIndex(["int32", "string", "float32", "colors"])
-    );
-
-    const dfA = sourceDf.isubsetMask(
-      new Uint8Array([0, 1, 1]),
-      new Uint8Array([1, 0, 0, 1])
-    );
-    expect(dfA.dims).toEqual([2, 2]);
-    expect(dfA.icol(0).asArray()).toEqual(new Int32Array([1, 2]));
-    expect(dfA.icol(1).asArray()).toEqual(["green", "blue"]);
-    expect(dfA.rowIndex.keys()).toEqual(new Int32Array([4, 6]));
-    expect(dfA.colIndex.keys()).toEqual(["int32", "colors"]);
-  });
-});
-
-describe("dataframe factories", () => {
-  test("create", () => {
-    const df = Dataframe.Dataframe.create(
-      [3, 3],
-      [
-        new Array(3).fill(0),
-        new Int16Array(3).fill(99),
-        new Float64Array(3).fill(1.1)
-      ]
-    );
-
-    expect(df).toBeDefined();
-    expect(df.dims).toEqual([3, 3]);
-    expect(df).toHaveLength(3);
-    expect(df.iat(0, 0)).toEqual(0);
-    expect(df.iat(1, 1)).toEqual(99);
-    expect(df.iat(2, 2)).toBeCloseTo(1.1);
-    expect(df.iat(0, 0)).toEqual(df.at(0, 0));
-    expect(df.iat(1, 1)).toEqual(df.at(1, 1));
-    expect(df.iat(2, 2)).toEqual(df.at(2, 2));
-  });
-
-  test("clone", () => {
-    const dfA = new Dataframe.Dataframe(
-      [3, 2],
-      [new Int32Array([0, 1, 2]), new Int32Array([3, 4, 5])],
-      new Dataframe.DenseInt32Index([2, 1, 0]),
-      new Dataframe.KeyIndex(["A", "B"])
-    );
-
-    const dfB = dfA.clone();
-    expect(dfB).not.toBe(dfA);
-    expect(dfB.dims).toEqual(dfA.dims);
-    expect(dfB).toHaveLength(dfA.length);
-    expect(dfB.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
-    expect(dfB.colIndex.keys()).toEqual(dfA.colIndex.keys());
-    for (let i = 0, l = dfB.dims[1]; i < l; i += 1) {
-      expect(dfB.icol(i).asArray()).toEqual(dfA.icol(i).asArray());
-    }
-  });
-
-  describe("withCol", () => {
-    test("KeyIndex", () => {
-      const df = new Dataframe.Dataframe(
-        [2, 2],
-        [["red", "blue"], [true, false]],
-        null,
-        new Dataframe.KeyIndex(["colors", "bools"])
-      );
-      const dfA = df.withCol("numbers", [1, 0]);
-
-      expect(dfA).toBeDefined();
-      expect(dfA.dims).toEqual([2, 3]);
-      expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]);
-      expect(dfA.icol(1).asArray()).toEqual([true, false]);
-      expect(dfA.icol(2).asArray()).toEqual([1, 0]);
-      expect(dfA.col("numbers").asArray()).toEqual([1, 0]);
-      expect(dfA.colIndex.keys()).toEqual(["colors", "bools", "numbers"]);
-      expect(df.colIndex.keys()).toEqual(["colors", "bools"]);
-      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
-    });
-
-    test("DenseInt32Index", () => {
-      const df = new Dataframe.Dataframe(
-        [2, 2],
-        [["red", "blue"], [true, false]],
-        null,
-        new Dataframe.DenseInt32Index([74, 75])
-      );
-      const dfA = df.withCol(72, [1, 0]);
-
-      expect(dfA).toBeDefined();
-      expect(dfA.dims).toEqual([2, 3]);
-      expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]);
-      expect(dfA.icol(1).asArray()).toEqual([true, false]);
-      expect(dfA.icol(2).asArray()).toEqual([1, 0]);
-      expect(dfA.col(74).asArray()).toEqual(["red", "blue"]);
-      expect(dfA.col(75).asArray()).toEqual([true, false]);
-      expect(dfA.col(72).asArray()).toEqual([1, 0]);
-      expect(dfA.colIndex.keys()).toEqual(new Int32Array([74, 75, 72]));
-      expect(df.colIndex.keys()).toEqual(new Int32Array([74, 75]));
-      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
-    });
-
-    test("DenseInt32Index promote", () => {
-      const df = new Dataframe.Dataframe(
-        [2, 2],
-        [["red", "blue"], [true, false]],
-        null,
-        new Dataframe.DenseInt32Index([74, 75])
-      );
-      const dfA = df.withCol(999, [1, 0]);
-
-      expect(dfA).toBeDefined();
-      expect(dfA.dims).toEqual([2, 3]);
-      expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]);
-      expect(dfA.icol(1).asArray()).toEqual([true, false]);
-      expect(dfA.icol(2).asArray()).toEqual([1, 0]);
-      expect(dfA.col(74).asArray()).toEqual(["red", "blue"]);
-      expect(dfA.col(75).asArray()).toEqual([true, false]);
-      expect(dfA.col(999).asArray()).toEqual([1, 0]);
-      expect(dfA.colIndex.keys()).toEqual(new Int32Array([74, 75, 999]));
-      expect(df.colIndex.keys()).toEqual(new Int32Array([74, 75]));
-      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
-    });
-
-    test("IdentityInt32Index with last", () => {
-      const df = new Dataframe.Dataframe(
-        [2, 2],
-        [["red", "blue"], [true, false]],
-        null,
-        null
-      );
-      const dfA = df.withCol(2, [1, 0]);
-
-      expect(dfA).toBeDefined();
-      expect(dfA.dims).toEqual([2, 3]);
-      expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]);
-      expect(dfA.icol(1).asArray()).toEqual([true, false]);
-      expect(dfA.icol(2).asArray()).toEqual([1, 0]);
-      expect(dfA.col(0).asArray()).toEqual(["red", "blue"]);
-      expect(dfA.col(1).asArray()).toEqual([true, false]);
-      expect(dfA.col(2).asArray()).toEqual([1, 0]);
-      expect(dfA.colIndex.keys()).toEqual(new Int32Array([0, 1, 2]));
-      expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1]));
-      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
-    });
-
-    test("IdentityInt32Index promote", () => {
-      const df = new Dataframe.Dataframe(
-        [2, 2],
-        [["red", "blue"], [true, false]],
-        null,
-        null
-      );
-      const dfA = df.withCol(99, [1, 0]);
-
-      expect(dfA).toBeDefined();
-      expect(dfA.dims).toEqual([2, 3]);
-      expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]);
-      expect(dfA.icol(1).asArray()).toEqual([true, false]);
-      expect(dfA.icol(2).asArray()).toEqual([1, 0]);
-      expect(dfA.col(0).asArray()).toEqual(["red", "blue"]);
-      expect(dfA.col(1).asArray()).toEqual([true, false]);
-      expect(dfA.col(99).asArray()).toEqual([1, 0]);
-      expect(dfA.colIndex.keys()).toEqual(new Int32Array([0, 1, 99]));
-      expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1]));
-      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
-    });
-
-    describe("handle column dimensions correctly", () => {
-      /*
-      there are two conditions:
-      - empty dataframe - will accept an add of any dimensionality
-      - non-empty dataframe - added column must match row-count dimension
-      */
-      test("empty.withCol", () => {
-        const edf = Dataframe.Dataframe.empty();
-        const df = edf.withCol("foo", [1, 2, 3]);
-
-        expect(edf).toBeDefined();
-        expect(df).toBeDefined();
-        expect(edf).not.toEqual(df);
-        expect(df.dims).toEqual([3, 1]);
-        expect(df.icol(0).asArray()).toEqual([1, 2, 3]);
-      });
-
-      test("withCol dimension check", () => {
-        const dfA = new Dataframe.Dataframe([1, 1], [["a"]]);
-        expect(() => {
-          dfA.withCol(1, []);
-        }).toThrow(RangeError);
-      });
-    });
-  });
-
-  describe("dropCol", () => {
-    test("KeyIndex", () => {
-      const df = new Dataframe.Dataframe(
-        [2, 3],
-        [["red", "blue"], [true, false], [1, 0]],
-        null,
-        new Dataframe.KeyIndex(["colors", "bools", "numbers"])
-      );
-      const dfA = df.dropCol("colors");
-
-      expect(dfA).toBeDefined();
-      expect(dfA.dims).toEqual([2, 2]);
-      expect(dfA.icol(0).asArray()).toEqual([true, false]);
-      expect(dfA.icol(1).asArray()).toEqual([1, 0]);
-      expect(dfA.col("numbers").asArray()).toEqual([1, 0]);
-      expect(dfA.colIndex.keys()).toEqual(["bools", "numbers"]);
-      expect(df.colIndex.keys()).toEqual(["colors", "bools", "numbers"]);
-      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
-    });
-
-    test("IdentityInt32Index drop first", () => {
-      const df = new Dataframe.Dataframe(
-        [2, 3],
-        [["red", "blue"], [true, false], [1, 0]],
-        null,
-        null
-      );
-      const dfA = df.dropCol(0);
-
-      expect(dfA).toBeDefined();
-      expect(dfA.dims).toEqual([2, 2]);
-      expect(dfA.icol(0).asArray()).toEqual([true, false]);
-      expect(dfA.icol(1).asArray()).toEqual([1, 0]);
-      expect(df.col(1).asArray()).toEqual(dfA.col(1).asArray());
-      expect(df.col(2).asArray()).toEqual(dfA.col(2).asArray());
-      expect(dfA.colIndex.keys()).toEqual(new Int32Array([1, 2]));
-      expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1, 2]));
-      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
-    });
-
-    test("IdentityInt32Index drop last", () => {
-      const df = new Dataframe.Dataframe(
-        [2, 3],
-        [["red", "blue"], [true, false], [1, 0]],
-        null,
-        null
-      );
-      const dfA = df.dropCol(2);
-
-      expect(dfA).toBeDefined();
-      expect(dfA.dims).toEqual([2, 2]);
-      expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]);
-      expect(dfA.icol(1).asArray()).toEqual([true, false]);
-      expect(df.col(0).asArray()).toEqual(dfA.col(0).asArray());
-      expect(df.col(1).asArray()).toEqual(dfA.col(1).asArray());
-      expect(dfA.colIndex.keys()).toEqual(new Int32Array([0, 1]));
-      expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1, 2]));
-      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
-    });
-
-    test("DenseInt32Index", () => {
-      const df = new Dataframe.Dataframe(
-        [2, 3],
-        [["red", "blue"], [true, false], [1, 0]],
-        null,
-        new Dataframe.DenseInt32Index([102, 101, 100])
-      );
-      const dfA = df.dropCol(101);
-
-      expect(dfA).toBeDefined();
-      expect(dfA.dims).toEqual([2, 2]);
-      expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]);
-      expect(dfA.icol(1).asArray()).toEqual([1, 0]);
-      expect(dfA.col(100).asArray()).toEqual([1, 0]);
-      expect(dfA.col(102).asArray()).toEqual(["red", "blue"]);
-      expect(dfA.colIndex.keys()).toEqual(new Int32Array([102, 100]));
-      expect(df.colIndex.keys()).toEqual(new Int32Array([102, 101, 100]));
-      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
-    });
-  });
-
-  describe("mapColumns", () => {
-    test("identity", () => {
-      const dfA = Dataframe.Dataframe.create(
-        [3, 3],
-        [
-          new Array(3).fill(0),
-          new Int16Array(3).fill(99),
-          new Float64Array(3).fill(1.1)
-        ]
-      );
-      const dfB = dfA.mapColumns((col, idx) => {
-        expect(dfA.icol(idx).asArray()).toBe(col);
-        return col;
-      });
-      expect(dfA).not.toBe(dfB);
-      expect(dfA.dims).toEqual(dfB.dims);
-      for (let c = 0; c < dfA.dims[1]; c += 1) {
-        expect(dfA.icol(c).asArray()).toBe(dfB.icol(c).asArray());
-      }
-    });
-
-    test("transform", () => {
-      const dfA = Dataframe.Dataframe.create(
-        [3, 3],
-        [new Array(3).fill(0), new Array(3).fill(0), new Array(3).fill(0)]
-      );
-      const dfB = dfA.mapColumns(() => {
-        return new Array(3).fill(1);
-      });
-      expect(dfA).not.toBe(dfB);
-      expect(dfB.iat(0, 0)).toEqual(1);
-      expect(dfB.iat(0, 1)).toEqual(1);
-      expect(dfB.iat(0, 2)).toEqual(1);
-    });
-  });
-});
-
-describe("dataframe col", () => {
-  let df = null;
-  beforeEach(() => {
-    df = new Dataframe.Dataframe(
-      [2, 2],
-      [[true, false], [1, 0]],
-      null,
-      new Dataframe.KeyIndex(["A", "B"])
-    );
-  });
-
-  test("col", () => {
-    expect(df).toBeDefined();
-    expect(df.col("A")).toBe(df.icol(0));
-    expect(df.col("B")).toBe(df.icol(1));
-    expect(df.col("undefined")).toBeUndefined();
-    expect(df.icol("undefined")).toBeUndefined();
-
-    const colA = df.col("A");
-    expect(colA).toBeInstanceOf(Function);
-    expect(colA.asArray).toBeInstanceOf(Function);
-    expect(colA.has).toBeInstanceOf(Function);
-    expect(colA.ihas).toBeInstanceOf(Function);
-    expect(colA.indexOf).toBeInstanceOf(Function);
-    expect(colA.iget).toBeInstanceOf(Function);
-  });
-
-  test("col.asArray", () => {
-    expect(df).toBeDefined();
-    expect(df.col("A").asArray()).toEqual([true, false]);
-    expect(df.icol(0).asArray()).toEqual([true, false]);
-    expect(df.col("B").asArray()).toEqual([1, 0]);
-    expect(df.icol(1).asArray()).toEqual([1, 0]);
-  });
-
-  test("col.has", () => {
-    expect(df).toBeDefined();
-    expect(df.col("A").has(-1)).toBe(false);
-    expect(df.col("A").has(0)).toBe(true);
-    expect(df.col("A").has(1)).toBe(true);
-    expect(df.col("A").has(2)).toBe(false);
-    expect(df.col("B").has(-1)).toBe(false);
-    expect(df.col("B").has(0)).toBe(true);
-    expect(df.col("B").has(1)).toBe(true);
-    expect(df.col("B").has(2)).toBe(false);
-  });
-
-  test("col.ihas", () => {
-    expect(df).toBeDefined();
-    expect(df.col("A").ihas(-1)).toBe(false);
-    expect(df.col("A").ihas(0)).toBe(true);
-    expect(df.col("A").ihas(1)).toBe(true);
-    expect(df.col("A").ihas(2)).toBe(false);
-    expect(df.col("B").ihas(-1)).toBe(false);
-    expect(df.col("B").ihas(0)).toBe(true);
-    expect(df.col("B").ihas(1)).toBe(true);
-    expect(df.col("B").ihas(2)).toBe(false);
-  });
-
-  test("col.iget", () => {
-    expect(df).toBeDefined();
-    expect(df.col("A").iget(0)).toEqual(df.iat(0, 0));
-    expect(df.col("B").iget(1)).toEqual(df.iat(1, 1));
-  });
-
-  test("col.indexOf", () => {
-    expect(df).toBeDefined();
-    expect(df.col("A").indexOf(true)).toEqual(0);
-    expect(df.col("A").indexOf(false)).toEqual(1);
-    expect(df.col("A").indexOf(99)).toBeUndefined();
-    expect(df.col("A").indexOf(undefined)).toBeUndefined();
-    expect(df.col("A").indexOf(1)).toBeUndefined();
-
-    expect(df.col("B").indexOf(1)).toEqual(0);
-    expect(df.col("B").indexOf(0)).toEqual(1);
-    expect(df.col("B").indexOf(99)).toBeUndefined();
-    expect(df.col("B").indexOf(undefined)).toBeUndefined();
-    expect(df.col("B").indexOf(true)).toBeUndefined();
-  });
-});

client/__tests__/util/dataframe/summarize.test.js

@@ -1,253 +0,0 @@
-import * as Dataframe from "../../../src/util/dataframe";
-
-function float32Conversion(f) {
-  return new Float32Array([f])[0];
-}
-
-describe("Dataframe column summary", () => {
-  test("empty column test", () => {
-    const df = Dataframe.Dataframe.create([0, 1], [[]]);
-    const summary = df.icol(0).summarize();
-    expect(summary).toEqual(
-      expect.objectContaining({
-        categorical: true,
-        categories: [],
-        categoryCounts: new Map(),
-        numCategories: 0
-      })
-    );
-  });
-
-  test("simple test", () => {
-    const df = new Dataframe.Dataframe(
-      [1, 6],
-      [
-        ["n1"],
-        ["hi"],
-        [true],
-        new Float32Array([39.3]),
-        new Int32Array([99]),
-        [1]
-      ],
-      null,
-      new Dataframe.KeyIndex([
-        "name",
-        "nameString",
-        "nameBoolean",
-        "nameFloat32",
-        "nameInt32",
-        "nameCategorical"
-      ])
-    );
-
-    expect(df.icol(0).summarize()).toEqual(
-      expect.objectContaining({
-        categorical: true,
-        categories: ["n1"],
-        categoryCounts: new Map([["n1", 1]]),
-        numCategories: 1
-      })
-    );
-    expect(df.icol(1).summarize()).toEqual(
-      expect.objectContaining({
-        categorical: true,
-        categories: ["hi"],
-        categoryCounts: new Map([["hi", 1]]),
-        numCategories: 1
-      })
-    );
-    expect(df.icol(2).summarize()).toEqual(
-      expect.objectContaining({
-        categorical: true,
-        categories: [true],
-        categoryCounts: new Map([[true, 1]]),
-        numCategories: 1
-      })
-    );
-    expect(df.icol(3).summarize()).toEqual(
-      expect.objectContaining({
-        categorical: false,
-        min: float32Conversion(39.3),
-        max: float32Conversion(39.3),
-        nan: 0,
-        ninf: 0,
-        pinf: 0
-      })
-    );
-    expect(df.icol(4).summarize()).toEqual(
-      expect.objectContaining({
-        categorical: false,
-        min: 99,
-        max: 99,
-        nan: 0,
-        ninf: 0,
-        pinf: 0
-      })
-    );
-    expect(df.icol(5).summarize()).toEqual(
-      expect.objectContaining({
-        categorical: true,
-        categories: [1],
-        categoryCounts: new Map([[1, 1]]),
-        numCategories: 1
-      })
-    );
-  });
-
-  test("multi test", () => {
-    const df = new Dataframe.Dataframe(
-      [3, 6],
-      [
-        ["n0", "n1", "n2"],
-        ["hi", "hi", "bye"],
-        [false, true, true],
-        new Float32Array([39.3, 39.3, 0]),
-        new Int32Array([99, 99, 99]),
-        [1, false, "0"]
-      ],
-      null,
-      new Dataframe.KeyIndex([
-        "name",
-        "nameString",
-        "nameBoolean",
-        "nameFloat32",
-        "nameInt32",
-        "nameCategorical"
-      ])
-    );
-
-    expect(df.icol(0).summarize()).toEqual(
-      expect.objectContaining({
-        categorical: true,
-        categories: expect.arrayContaining(["n0", "n1", "n2"]),
-        categoryCounts: new Map([["n0", 1], ["n1", 1], ["n2", 1]]),
-        numCategories: 3
-      })
-    );
-    expect(df.icol(1).summarize()).toEqual(
-      expect.objectContaining({
-        categorical: true,
-        categories: expect.arrayContaining(["hi", "bye"]),
-        categoryCounts: new Map([["hi", 2], ["bye", 1]]),
-        numCategories: 2
-      })
-    );
-    expect(df.icol(2).summarize()).toEqual(
-      expect.objectContaining({
-        categorical: true,
-        categories: expect.arrayContaining([true, false]),
-        categoryCounts: new Map([[true, 2], [false, 1]]),
-        numCategories: 2
-      })
-    );
-    expect(df.icol(3).summarize()).toEqual(
-      expect.objectContaining({
-        categorical: false,
-        min: 0,
-        max: float32Conversion(39.3),
-        nan: 0,
-        ninf: 0,
-        pinf: 0
-      })
-    );
-    expect(df.icol(4).summarize()).toEqual(
-      expect.objectContaining({
-        categorical: false,
-        min: 99,
-        max: 99,
-        nan: 0,
-        ninf: 0,
-        pinf: 0
-      })
-    );
-    expect(df.icol(5).summarize()).toEqual(
-      expect.objectContaining({
-        categorical: true,
-        categories: expect.arrayContaining([1, false, "0"]),
-        categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]),
-        numCategories: 3
-      })
-    );
-  });
-
-  test("non-finite numbers", () => {
-    const df = new Dataframe.Dataframe(
-      [4, 6],
-      [
-        ["n0", "n1", "n2", "n2"],
-        ["hi", "hi", "bye", "bye"],
-        [false, true, true, true],
-        new Float32Array([
-          39.3,
-          Number.NEGATIVE_INFINITY,
-          Number.NaN,
-          Number.POSITIVE_INFINITY
-        ]),
-        new Int32Array([99, 99, 99, 99]),
-        [1, false, "0", "0"]
-      ],
-      null,
-      new Dataframe.KeyIndex([
-        "name",
-        "nameString",
-        "nameBoolean",
-        "nameFloat32",
-        "nameInt32",
-        "nameCategorical"
-      ])
-    );
-
-    expect(df.icol(0).summarize()).toEqual(
-      expect.objectContaining({
-        categorical: true,
-        categories: expect.arrayContaining(["n0", "n1", "n2"]),
-        categoryCounts: new Map([["n0", 1], ["n1", 1], ["n2", 2]]),
-        numCategories: 3
-      })
-    );
-    expect(df.icol(1).summarize()).toEqual(
-      expect.objectContaining({
-        categorical: true,
-        categories: expect.arrayContaining(["hi", "bye"]),
-        categoryCounts: new Map([["hi", 2], ["bye", 1]]),
-        numCategories: 2
-      })
-    );
-    expect(df.icol(2).summarize()).toEqual(
-      expect.objectContaining({
-        categorical: true,
-        categories: expect.arrayContaining([true, false]),
-        categoryCounts: new Map([[true, 2], [false, 1]]),
-        numCategories: 2
-      })
-    );
-    expect(df.icol(3).summarize()).toEqual(
-      expect.objectContaining({
-        categorical: false,
-        min: float32Conversion(39.3),
-        max: float32Conversion(39.3),
-        nan: 1,
-        ninf: 1,
-        pinf: 1
-      })
-    );
-    expect(df.icol(4).summarize()).toEqual(
-      expect.objectContaining({
-        categorical: false,
-        min: 99,
-        max: 99,
-        nan: 0,
-        ninf: 0,
-        pinf: 0
-      })
-    );
-    expect(df.icol(5).summarize()).toEqual(
-      expect.objectContaining({
-        categorical: true,
-        categories: expect.arrayContaining([1, false, "0"]),
-        categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]),
-        numCategories: 3
-      })
-    );
-  });
-});

client/__tests__/util/quantile.test.js

@@ -1,29 +0,0 @@
-import quantile from "../../src/util/quantile";
-
-describe("quantile", () => {
-	test("single q", () => {
-		const arr = new Float32Array([9, 3, 5, 6, 0]);
-		expect(quantile([1.0], arr)).toMatchObject([9]);
-		expect(quantile([0.9], arr)).toMatchObject([9]);
-		expect(quantile([0.8], arr)).toMatchObject([9]);
-		expect(quantile([0.7], arr)).toMatchObject([6]);
-		expect(quantile([0.6], arr)).toMatchObject([6]);
-		expect(quantile([0.5], arr)).toMatchObject([5]);
-		expect(quantile([0.4], arr)).toMatchObject([5]);
-		expect(quantile([0.3], arr)).toMatchObject([3]);
-		expect(quantile([0.2], arr)).toMatchObject([3]);
-		expect(quantile([0.1], arr)).toMatchObject([0]);
-		expect(quantile([0], arr)).toMatchObject([0]);
-	});
-
-	test("multi q", () => {
-		const arr = new Float32Array([9, 3, 5, 6, 0]);
-		expect(quantile([0, 0.25, 0.5, 0.75, 1.0], arr)).toMatchObject([
-			0,
-			3,
-			5,
-			6,
-			9
-		]);
-	});
-});

client/__tests__/util/range.test.js

@@ -1,42 +0,0 @@
-import { range, rangeFill } from "../../src/util/range";
-
-describe("range", () => {
-	test("no defaults", () => {
-		expect(range(0, 3, 1)).toMatchObject([0, 1, 2]);
-	});
-
-	test("range(stop)", () => {
-		expect(range(3)).toMatchObject([0, 1, 2]);
-		expect(range(0)).toMatchObject([]);
-		expect(range(1)).toMatchObject([0]);
-	});
-
-	test("range(start,stop)", () => {
-		expect(range(0, 0)).toMatchObject([]);
-		expect(range(0, 2)).toMatchObject([0, 1]);
-		expect(range(4, 8)).toMatchObject([4, 5, 6, 7]);
-	});
-
-	test("range(start, stop, step", () => {
-		expect(range(4, 0, -1)).toMatchObject([4, 3, 2, 1]);
-		expect(range(0, 4, 2)).toMatchObject([0, 2]);
-	});
-});
-
-describe("rangefill", () => {
-	test("rangeFill(arr)", () => {
-		expect(rangeFill(new Int32Array(3))).toMatchObject(
-			new Int32Array([0, 1, 2])
-		);
-	});
-	test("rangeFill(arr, start)", () => {
-		expect(rangeFill(new Int32Array(2), 1)).toMatchObject(
-			new Int32Array([1, 2])
-		);
-	});
-	test("rangeFill(arr, start, step)", () => {
-		expect(rangeFill(new Int32Array(3), 2, -1)).toMatchObject(
-			new Int32Array([2, 1, 0])
-		);
-	});
-});

client/__tests__/util/stateManager/keyvalcache.test.js

@@ -0,0 +1,249 @@
+import _ from "lodash";
+import * as kvCache from "../../../src/util/stateManager/keyvalcache";
+
+/*
+This is PRIVATE to keyvalcache and must be kept in sync with
+any changs ot that module.  Need to Know - to enable error handling test
+*/
+const cachePrivateKey = "__kvcachekey__";
+
+/*
+helper function - promisify setTimeout()
+*/
+function timeout(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+describe("kvcache API", () => {
+  /*
+  test the happy path create/set/get API
+  */
+
+  test("simple create", () => {
+    /* with defaults */
+    const kvc = kvCache.create();
+    expect(kvc).toBeDefined();
+    expect(kvc).toEqual(expect.objectContaining({}));
+    expect(kvCache.get(kvc, "test")).toBeUndefined();
+
+    /* with params */
+    const kvc1 = kvCache.create(/* lowWatermark */ 99, /* minTTL */ 0);
+    expect(kvc1).toBeDefined();
+    expect(kvc1).toEqual(expect.objectContaining({}));
+  });
+
+  test("set/get", () => {
+    /*
+    - check basic get/set functionality
+    - check set does not mutate source cache
+    */
+    const keyName = "foo";
+    const kvc1 = kvCache.create();
+    expect(kvc1).toBeDefined();
+    expect(kvCache.get(kvc1, keyName)).toBeUndefined();
+
+    const val2 = [2];
+    const kvc2 = kvCache.set(kvc1, keyName, val2);
+    expect(kvc2).toBeDefined();
+    expect(kvc2).not.toBe(kvc1);
+    expect(kvCache.get(kvc1, keyName)).toBeUndefined();
+    expect(kvCache.get(kvc2, keyName)).toBe(val2);
+
+    const val3 = [3];
+    const kvc3 = kvCache.set(kvc2, keyName, val3);
+    expect(kvc3).toBeDefined();
+    expect(kvc3).not.toBe(kvc1);
+    expect(kvc3).not.toBe(kvc2);
+    expect(kvCache.get(kvc1, keyName)).toBeUndefined();
+    expect(kvCache.get(kvc2, keyName)).toBe(val2);
+    expect(kvCache.get(kvc3, keyName)).toBe(val3);
+  });
+});
+
+describe("common error handling", () => {
+  /*
+  Test common error handlers
+  */
+
+  test("set() protection from namespace pollution", () => {
+    /*
+    Test that set() will not allow use of the private cache key
+    */
+    const kvc = kvCache.create();
+    expect(() => {
+      kvCache.set(kvc, cachePrivateKey, {});
+    }).toThrow();
+  });
+
+  test("create() does not accept bogus config", () => {
+    expect(() => {
+      kvCache.create([], {});
+    }).toThrow();
+    expect(() => {
+      kvCache.create(-99, 0);
+    }).toThrow();
+    expect(() => {
+      kvCache.create(100, -1);
+    }).toThrow();
+    expect(() => {
+      kvCache.create(1000, "foobar");
+    }).toThrow();
+    expect(() => {
+      kvCache.create(null, 8);
+    }).toThrow();
+  });
+});
+
+describe("map", () => {
+  /*
+  Test kvCache.map() - create new cache that is a transformation of an
+  existing cache
+  */
+  test("map of empty cache", () => {
+    const kvc = kvCache.create();
+    const callback = jest.fn();
+    const kvcMapped = kvCache.map(kvc, callback);
+    expect(callback).not.toHaveBeenCalled();
+    expect(kvcMapped).toBeDefined();
+    expect(kvcMapped).not.toBe(kvc); // immutable operation
+    expect(kvcMapped).toEqual(kvc);
+  });
+
+  test("map of non-empty cache", () => {
+    const key = "aKey";
+    const val = [0, 1, 2];
+    let kvc = kvCache.create();
+    kvc = kvCache.set(kvc, key, val);
+    const mockCB = jest.fn().mockImplementation(v => [...v]);
+    const kvcMapped = kvCache.map(kvc, mockCB);
+
+    expect(kvcMapped).toBeDefined();
+    expect(kvcMapped).not.toBe(kvc); // immutable operation
+    expect(_.isEqual(kvc, kvcMapped)).toBe(true);
+
+    expect(mockCB).toHaveBeenCalledTimes(1);
+    expect(mockCB).toHaveBeenLastCalledWith(val, key);
+  });
+});
+
+describe("flush", () => {
+  /*
+  test various cache flush behavior
+  */
+  test("flush - lowWatermark, disable minTTL", () => {
+    /*
+    verify lowWatermark functions correctly
+    */
+
+    // set lowWatermark to 2, set three times - only the final two
+    // should remain.
+    let kvc = kvCache.create(2, 0);
+    ["a", "b", "c"].forEach(k => {
+      kvc = kvCache.set(kvc, k, []);
+    });
+
+    expect(kvc).toEqual(
+      expect.objectContaining({
+        b: expect.arrayContaining([]),
+        c: expect.arrayContaining([])
+      })
+    );
+    expect(kvc).toEqual(
+      expect.not.objectContaining({
+        a: expect.arrayContaining([])
+      })
+    );
+  });
+
+  test("flush - minTTL, disable lowWatermark", async () => {
+    /*
+    verify minTTL functions correctly
+    */
+
+    // set minTTL to 1 ms
+    let kvc = kvCache.create(0, 10);
+    kvc = kvCache.set(kvc, "a", []);
+    await timeout(20);
+    ["b", "c"].forEach(k => {
+      kvc = kvCache.set(kvc, k, []);
+    });
+
+    expect(kvc).toEqual(
+      expect.objectContaining({
+        b: expect.arrayContaining([]),
+        c: expect.arrayContaining([])
+      })
+    );
+    expect(kvc).toEqual(
+      expect.not.objectContaining({
+        a: expect.arrayContaining([])
+      })
+    );
+  });
+
+  test("flush - minTTL and lowWatermark", async () => {
+    /*
+    verify minTTL functions correctly
+    */
+
+    // set lowwatermark to 3, minTTL to 1 ms
+    let kvc = kvCache.create(3, 10);
+    kvc = kvCache.set(kvc, "a", []);
+    // delay
+    await timeout(20);
+    ["b", "c"].forEach(k => {
+      kvc = kvCache.set(kvc, k, []);
+    });
+
+    expect(kvc).toEqual(
+      expect.objectContaining({
+        a: expect.arrayContaining([]),
+        b: expect.arrayContaining([]),
+        c: expect.arrayContaining([])
+      })
+    );
+
+    kvc = kvCache.set(kvc, "d", []);
+    expect(kvc).toEqual(
+      expect.objectContaining({
+        b: expect.arrayContaining([]),
+        c: expect.arrayContaining([]),
+        d: expect.arrayContaining([])
+      })
+    );
+    expect(kvc).toEqual(
+      expect.not.objectContaining({
+        a: expect.arrayContaining([])
+      })
+    );
+  });
+
+  test("manual flush", async () => {
+    let kvc = kvCache.create(1, 10);
+    ["a", "b", "c", "d"].forEach(k => {
+      kvc = kvCache.set(kvc, k, []);
+    });
+
+    // Before TTL has expired, should have all values in cache.
+    expect(kvc).toEqual(
+      expect.objectContaining({
+        a: expect.arrayContaining([]),
+        b: expect.arrayContaining([]),
+        c: expect.arrayContaining([])
+      })
+    );
+
+    // let TTL expire
+    await timeout(10);
+
+    // manually flush
+    const postFlushKvc = kvCache.flush(kvc);
+    expect(postFlushKvc).toBeDefined();
+    expect(postFlushKvc).not.toBe(kvc);
+    expect(postFlushKvc).toEqual(
+      expect.objectContaining({
+        d: expect.arrayContaining([])
+      })
+    );
+  });
+});

client/__tests__/util/stateManager/sampleResponses.js

@@ -1,7 +1,5 @@
 /* eslint no-bitwise: "off" */
 import _ from "lodash";
-import { flatbuffers } from "flatbuffers";
-import { NetEncoding } from "../../../src/util/stateManager/matrix_generated";
 
 /*
 test data mocking REST 0.2 API responses.  Used in several tests.
@@ -60,7 +58,7 @@ const aSchemaResponse = {
   }
 };
 
-const anAnnotationsObsJSONResponse = {
+const anAnnotationsObsResponse = {
   names: ["name", "field1", "field2", "field3", "field4"],
   data: _()
     .range(nObs)
@@ -75,7 +73,7 @@ const anAnnotationsObsJSONResponse = {
     .value()
 };
 
-const anAnnotationsVarJSONResponse = {
+const anAnnotationsVarResponse = {
   names: ["fieldA", "fieldB", "fieldC", "fieldD", "name"],
   data: _()
     .range(nVar)
@@ -90,80 +88,15 @@ const anAnnotationsVarJSONResponse = {
     .value()
 };
 
-function encodeTypedArray(builder, uType, uData) {
+const aLayoutResponse = {
-  const uTypeName = NetEncoding.TypedArray[uType];
+  layout: {
-  const ArrayType = NetEncoding[uTypeName];
+    ndims: 2,
-  const dv = ArrayType.createDataVector(builder, uData);
+    coordinates: _()
-  builder.startObject(1);
+      .range(nObs)
-  builder.addFieldOffset(0, dv, 0);
+      .map(idx => [idx, Math.random(), Math.random()])
-  return builder.endObject();
+      .value()
-}
-
-function encodeMatrix(columns, colIndex = undefined) {
-  const utf8Encoder = new TextEncoder("utf-8");
-  const builder = new flatbuffers.Builder(1024);
-  const cols = _.map(columns, carr => {
-    let uType;
-    let tarr;
-    if (_.every(carr, _.isNumber)) {
-      uType = NetEncoding.TypedArray.Float32Array;
-      tarr = encodeTypedArray(builder, uType, new Float32Array(carr));
-    } else {
-      uType = NetEncoding.TypedArray.JSONEncodedArray;
-      const json = JSON.stringify(carr);
-      const jsonUTF8 = utf8Encoder.encode(json);
-      tarr = encodeTypedArray(builder, uType, jsonUTF8);
-    }
-    NetEncoding.Column.startColumn(builder);
-    NetEncoding.Column.addUType(builder, uType);
-    NetEncoding.Column.addU(builder, tarr);
-    return NetEncoding.Column.endColumn(builder);
-  });
-
-  const encColumns = NetEncoding.Matrix.createColumnsVector(builder, cols);
-
-  let encColIndex;
-  if (colIndex) {
-    encColIndex = encodeTypedArray(
-      builder,
-      NetEncoding.TypedArray.JSONEncodedArray,
-      utf8Encoder.encode(JSON.stringify(colIndex))
-    );
   }
-
+};
-  NetEncoding.Matrix.startMatrix(builder);
-  NetEncoding.Matrix.addNRows(builder, columns[0].length);
-  NetEncoding.Matrix.addNCols(builder, columns.length);
-  NetEncoding.Matrix.addColumns(builder, encColumns);
-  if (colIndex) {
-    NetEncoding.Matrix.addColIndexType(
-      builder,
-      NetEncoding.TypedArray.JSONEncodedArray
-    );
-    NetEncoding.Matrix.addColIndex(builder, encColIndex);
-  }
-  const root = NetEncoding.Matrix.endMatrix(builder);
-  builder.finish(root);
-  return builder.asUint8Array();
-}
-
-const anAnnotationsObsFBSResponse = (() => {
-  const columns = _.zip(...anAnnotationsObsJSONResponse.data).slice(1);
-  return encodeMatrix(columns, anAnnotationsObsJSONResponse.names);
-})();
-
-const anAnnotationsVarFBSResponse = (() => {
-  const columns = _.zip(...anAnnotationsVarJSONResponse.data).slice(1);
-  return encodeMatrix(columns, anAnnotationsVarJSONResponse.names);
-})();
-
-const aLayoutFBSResponse = (() => {
-  const coords = [
-    new Float32Array(nObs).fill(Math.random()),
-    new Float32Array(nObs).fill(Math.random())
-  ];
-  return encodeMatrix(coords, ["umap_0", "umap_1"]);
-})();
 
 const aDataObsResponse = {
   var: [2, 4, 29],
@@ -174,10 +107,10 @@ const aDataObsResponse = {
 };
 
 export {
-  aLayoutFBSResponse as layoutObs,
+  aLayoutResponse as layoutObs,
   aDataObsResponse as dataObs,
-  anAnnotationsVarFBSResponse as annotationsVar,
+  anAnnotationsVarResponse as annotationsVar,
-  anAnnotationsObsFBSResponse as annotationsObs,
+  anAnnotationsObsResponse as annotationsObs,
   aSchemaResponse as schema,
   aConfigResponse as config
 };

client/__tests__/util/stateManager/universe.test.js

@@ -1,13 +1,13 @@
+import _ from "lodash";
 import * as Universe from "../../../src/util/stateManager/universe";
-import * as Dataframe from "../../../src/util/dataframe";
 import * as REST from "./sampleResponses";
 
-describe("createUniverseFromResponse", () => {
+describe("createUniverseFromRestV02Response", () => {
   /*
-  test createUniverseFromResponse - this function converts
+  test createUniverseFromRestV02Response - this function converts
   a set of REST 0.2 responses into a "new" Universe.
 
-  createUniverseFromResponse(
+  createUniverseFromRestV02Response(
     configResponse,
     schemaResponse,
     annotationsObsResponse,
@@ -30,7 +30,8 @@ describe("createUniverseFromResponse", () => {
     create a universe from sample data nad validate its shape & contents
     */
     const { nObs, nVar } = REST.schema.schema.dataframe;
-    const universe = Universe.createUniverseFromResponse(
+
+    const universe = Universe.createUniverseFromRestV02Response(
       REST.config,
       REST.schema,
       REST.annotationsObs,
@@ -41,26 +42,80 @@ describe("createUniverseFromResponse", () => {
     expect(universe).toBeDefined();
     expect(universe).toMatchObject(
       expect.objectContaining({
+        api: "0.2",
         nObs,
         nVar,
         schema: REST.schema.schema,
-        obsAnnotations: expect.any(Dataframe.Dataframe),
+        obsAnnotations: expect.any(Array),
-        varAnnotations: expect.any(Dataframe.Dataframe),
+        varAnnotations: expect.any(Array),
-        obsLayout: expect.any(Dataframe.Dataframe),
+        obsNameToIndexMap: expect.any(Object),
-        varData: expect.any(Dataframe.Dataframe)
+        varNameToIndexMap: expect.any(Object),
+        obsLayout: expect.objectContaining({
+          X: expect.any(Float32Array),
+          Y: expect.any(Float32Array)
+        }),
+        varDataCache: expect.any(Object)
       })
     );
 
-    expect(universe.obsAnnotations.dims).toEqual([
+    expect(universe.obsAnnotations).toHaveLength(nObs);
-      nObs,
+    expect(_.keys(universe.obsNameToIndexMap)).toHaveLength(nObs);
-      REST.schema.schema.annotations.obs.length
+    expect(universe.obsLayout.X).toHaveLength(nObs);
-    ]);
+    expect(universe.obsLayout.Y).toHaveLength(nObs);
-    expect(universe.obsLayout.dims).toEqual([nObs, 2]);
+    expect(universe.varAnnotations).toHaveLength(nVar);
-    expect(universe.obsLayout.colIndex.keys()).toEqual(["X", "Y"]);
+    expect(_.keys(universe.varNameToIndexMap)).toHaveLength(nVar);
-    expect(universe.varAnnotations.dims).toEqual([
+  });
-      nVar,
+});
-      REST.schema.schema.annotations.var.length
+
-    ]);
+describe("convertExpressionRESTv02ToObject", () => {
-    expect(universe.varData.isEmpty()).toBeTruthy();
+  /*
+  test convertExpressionRESTv02ToObject
+
+  convertExpressionRESTv02ToObject(
+    universe,
+    response) --> { geneName: Float32Array, geneName: Float32Array, ... }
+
+  reponse is a /data/obs response:
+  {
+    var: [ varIndices fetched ],
+    obs: [
+      [ obsIndex, evalue, ... ],
+      ...
+    ]
+  }
+  */
+  test("create from response data", () => {
+    const universe = Universe.createUniverseFromRestV02Response(
+      REST.config,
+      REST.schema,
+      REST.annotationsObs,
+      REST.annotationsVar,
+      REST.layoutObs
+    );
+    const expression = Universe.convertExpressionRESTv02ToObject(
+      universe,
+      REST.dataObs
+    );
+
+    /* Check that the expected keys are present */
+    const expectedGeneNames = _.map(
+      REST.dataObs.var,
+      v => REST.annotationsVar.data[v][5]
+    );
+    expect(Object.keys(expression)).toEqual(
+      expect.arrayContaining(expectedGeneNames)
+    );
+
+    const expectedExpressionValues = _.map(
+      _.unzip(REST.dataObs.obs),
+      a => new Float32Array(a)
+    );
+
+    _.forEach(REST.dataObs.var, (varIdx, idx) => {
+      const varName = universe.varAnnotations[varIdx].name;
+      expect(varName).toBeDefined();
+      expect(varIdx).toBe(universe.varNameToIndexMap[varName]);
+      expect(expression[varName]).toEqual(expectedExpressionValues[idx + 1]);
+    });
   });
 });

client/__tests__/util/stateManager/world.test.js

@@ -1,14 +1,13 @@
 import _ from "lodash";
 import * as Universe from "../../../src/util/stateManager/universe";
 import * as World from "../../../src/util/stateManager/world";
-import * as Dataframe from "../../../src/util/dataframe";
 import Crossfilter from "../../../src/util/typedCrossfilter";
-import { DimTypes } from "../../../src/util/typedCrossfilter/crossfilter";
 import * as REST from "./sampleResponses";
 import {
   obsAnnoDimensionName,
   layoutDimensionName
 } from "../../../src/util/nameCreators";
+import * as kvCache from "../../../src/util/stateManager/keyvalcache";
 
 /*
 Helper - creates universe, world, corssfilter and dimensionMap from
@@ -17,7 +16,7 @@ the default REST test response.
 const defaultBigBang = () => {
   /* create unverse, world, crossfilter and dimensionMap */
   /* create universe */
-  const universe = Universe.createUniverseFromResponse(
+  const universe = Universe.createUniverseFromRestV02Response(
     REST.config,
     REST.schema,
     REST.annotationsObs,
@@ -27,21 +26,21 @@ const defaultBigBang = () => {
   /* create world */
   const world = World.createWorldFromEntireUniverse(universe);
   /* create crossfilter */
-  const crossfilter = World.createObsDimensions(
+  const crossfilter = Crossfilter(world.obsAnnotations);
-    new Crossfilter(world.obsAnnotations),
+  /* create dimension map */
-    world
+  const dimensionMap = World.createObsDimensionMap(crossfilter, world);
-  );
 
   return {
     universe,
     world,
-    crossfilter
+    crossfilter,
+    dimensionMap
   };
 };
 
 describe("createWorldFromEntireUniverse", () => {
   test("create from REST sample", () => {
-    const universe = Universe.createUniverseFromResponse(
+    const universe = Universe.createUniverseFromRestV02Response(
       REST.config,
       REST.schema,
       REST.annotationsObs,
@@ -55,18 +54,31 @@ describe("createWorldFromEntireUniverse", () => {
 
     expect(world).toMatchObject(
       expect.objectContaining({
+        api: "0.2",
         nObs: universe.nObs,
         nVar: universe.nVar,
         schema: universe.schema,
-        obsAnnotations: expect.any(Dataframe.Dataframe),
+        obsAnnotations: universe.obsAnnotations,
-        varAnnotations: expect.any(Dataframe.Dataframe),
+        varAnnotations: universe.varAnnotations,
-        obsLayout: expect.any(Dataframe.Dataframe),
+        obsLayout: universe.obsLayout,
-        varData: expect.any(Dataframe.Dataframe),
+
-        clipQuantiles: { min: 0, max: 1 },
+        summary: expect.objectContaining({
-        unclipped: {
+          obs: _(REST.schema.schema.annotations.obs)
-          obsAnnotations: expect.any(Dataframe.Dataframe),
+            .filter(v => v.name !== "name")
-          varData: expect.any(Dataframe.Dataframe)
+            .keyBy("name")
-        }
+            .mapValues(() => expect.any(Object))
+            .value(),
+          var: _(REST.schema.schema.annotations.var)
+            .filter(v => v.name !== "name")
+            .keyBy("name")
+            .mapValues(() => expect.any(Object))
+            .value()
+        }),
+
+        varDataCache: expect.any(Object),
+
+        obsIndex: null, // null indicating full universe
+        obsBackIndex: null
       })
     );
   });
@@ -77,68 +89,73 @@ describe("createWorldFromCurrentSelection", () => {
     const {
       universe,
       world: originalWorld,
-      crossfilter: originalCrossfilter
+      crossfilter,
+      dimensionMap
     } = defaultBigBang();
 
     /* mock a selection */
-    const crossfilter = originalCrossfilter
+    dimensionMap[obsAnnoDimensionName("field1")].filterRange([0, 5]);
-      .select(obsAnnoDimensionName("field1"), { mode: "range", lo: 0, hi: 5 })
+    dimensionMap[obsAnnoDimensionName("field3")].filterExact(false);
-      .select(obsAnnoDimensionName("field3"), {
-        mode: "exact",
-        values: [false]
-      });
 
     /* create the world from the selection */
-    const world = World.createWorldBySelection(
+    const world = World.createWorldFromCurrentSelection(
       universe,
       originalWorld,
       crossfilter
     );
     expect(world).toBeDefined();
-    expect(world.nObs).toEqual(crossfilter.countSelected());
+    expect(world.nObs).toEqual(crossfilter.countFiltered());
 
     /*
     calculate expected values and match against result
     */
 
     /* matchFilter must match the dimension filters above */
-    const matchFilter = (df, row) => {
+    const matchFilter = val => val.field1 >= 0 && val.field1 < 5 && !val.field3;
-      const field1 = df.at(row, "field1");
+    const universeIndices = _()
-      const field3 = df.at(row, "field3");
-      return field1 >= 0 && field1 < 5 && !field3;
-    };
-    const matchingIndices = _()
       .range(universe.nObs)
-      .filter(idx => matchFilter(universe.obsAnnotations, idx))
+      .filter(idx => matchFilter(universe.obsAnnotations[idx]))
       .value();
 
+    const expected = {
+      nObs: universeIndices.length,
+      obsAnnotations: _.map(universeIndices, i => universe.obsAnnotations[i]),
+      obsLayout: {
+        X: new Float32Array(
+          _.map(universeIndices, i => universe.obsLayout.X[i])
+        ),
+        Y: new Float32Array(
+          _.map(universeIndices, i => universe.obsLayout.Y[i])
+        )
+      },
+      obsBackIndex: _.transform(
+        universeIndices,
+        (result, univIdx, worldIdx) => {
+          result[univIdx] = worldIdx;
+        },
+        new Uint32Array(universe.nObs).fill(-1)
+      ),
+      obsIndex: new Uint32Array(universeIndices)
+    };
+
     expect(world).toMatchObject(
       expect.objectContaining({
-        nObs: matchingIndices.length,
+        api: "0.2",
+        nObs: expected.nObs,
         nVar: universe.nVar,
         schema: universe.schema,
-        clipQuantiles: { min: 0, max: 1 },
+        obsAnnotations: expected.obsAnnotations,
-        obsAnnotations: expect.any(Dataframe.Dataframe),
+        varAnnotations: universe.varAnnotations,
-        varAnnotations: expect.any(Dataframe.Dataframe),
+        obsLayout: expected.obsLayout,
-        obsLayout: expect.any(Dataframe.Dataframe),
+        summary: {
-        varData: expect.any(Dataframe.Dataframe),
+          obs: expect.any(Object) /* we could do better! */,
-        unclipped: {
+          var: expect.any(Object) /* we could do better! */
-          obsAnnotations: expect.any(Dataframe.Dataframe),
+        },
-          varData: expect.any(Dataframe.Dataframe)
+        varDataCache: expect.any(Object),
-        }
+        obsIndex: expected.obsIndex,
+        obsBackIndex: expected.obsBackIndex
       })
     );
-
-    expect(world.obsAnnotations.rowIndex.keys()).toEqual(
-      new Int32Array(matchingIndices)
-    );
-    expect(world.obsAnnotations.colIndex.keys()).toEqual(
-      universe.obsAnnotations.colIndex.keys()
-    );
-    expect(world.obsLayout.rowIndex.keys()).toEqual(
-      new Int32Array(matchingIndices)
-    );
-    expect(world.obsLayout.colIndex.keys()).toEqual(["X", "Y"]);
   });
 });
 
@@ -150,32 +167,87 @@ describe("createObsDimensionMap", () => {
     - check that dimension typing is sane
     */
 
-    const { crossfilter } = defaultBigBang();
+    const { dimensionMap } = defaultBigBang();
-    const annotationNames = _.map(
+
-      REST.schema.schema.annotations.obs,
-      c => c.name
-    );
     const schemaByObsName = _.keyBy(REST.schema.schema.annotations.obs, "name");
-    expect(crossfilter).toBeDefined();
+    expect(dimensionMap).toBeDefined();
-    annotationNames.forEach(name => {
+    REST.annotationsObs.names.forEach(name => {
-      const dim = crossfilter.dimensions[obsAnnoDimensionName(name)];
+      const dim = dimensionMap[obsAnnoDimensionName(name)];
       if (name === "name") {
         expect(dim).toBeUndefined();
       } else {
         const { type } = schemaByObsName[name];
         if (type === "string" || type === "boolean" || type === "categorical") {
-          expect(dim.dim).toBeInstanceOf(DimTypes.enum);
+          expect(dim).toBeInstanceOf(Crossfilter.EnumDimension);
         } else {
-          expect(dim.dim).toBeInstanceOf(DimTypes.scalar);
+          expect(dim).toBeInstanceOf(Crossfilter.ScalarDimension);
         }
       }
     });
-    expect(
+    expect(dimensionMap[layoutDimensionName("X")]).toBeInstanceOf(
-      crossfilter.dimensions[layoutDimensionName("XY")].dim
+      Crossfilter.ScalarDimension
-    ).toBeInstanceOf(DimTypes.spatial);
+    );
+    expect(dimensionMap[layoutDimensionName("Y")]).toBeInstanceOf(
+      Crossfilter.ScalarDimension
+    );
   });
 });
 
+describe("subsetVarData", () => {
+  test("when world eq universe", () => {
+    const { universe, world } = defaultBigBang();
+    /* create a mock varData array for subsetting */
+    const sourceVarData = new Float32Array(universe.nObs);
+
+    /* expect literally the same object back */
+    const result = World.subsetVarData(world, universe, sourceVarData);
+    expect(result).toBe(sourceVarData);
+  });
+
+  test("when world neq universe", () => {
+    const { universe, world, crossfilter, dimensionMap } = defaultBigBang();
+    /* create a mock varData array for subsetting */
+    const sourceVarData = Float32Array.from(_.range(universe.nObs));
+
+    /* mock a selection */
+    dimensionMap[obsAnnoDimensionName("field1")].filterRange([0, 5]);
+    dimensionMap[obsAnnoDimensionName("field3")].filterExact(false);
+
+    /* create the world from the selection */
+    const newWorld = World.createWorldFromCurrentSelection(
+      universe,
+      world,
+      crossfilter
+    );
+    expect(newWorld.obsIndex).toMatchObject(new Uint32Array([0, 2]));
+
+    /* expect a subset */
+    const result = World.subsetVarData(newWorld, universe, sourceVarData);
+    expect(result).not.toBe(sourceVarData);
+    expect(result).toHaveLength(newWorld.nObs);
+    /* check that we have expected source var content */
+    expect(result).toMatchObject(new Float32Array([0, 2]));
+  });
+});
+
+describe("createVarDimension", () => {
+  /* create default universe */
+  const { world, crossfilter } = defaultBigBang();
+  /* create a mock var data cache */
+  const varDataCache = kvCache.set(
+    kvCache.create(),
+    "GENE",
+    Float32Array.from(_.range(world.nObs))
+  );
+  const result = World.createVarDimension(
+    world,
+    varDataCache,
+    crossfilter,
+    "GENE"
+  );
+  expect(result).toBeInstanceOf(Crossfilter.ScalarDimension);
+});
+
 describe("worldEqUniverse", () => {
   const { universe, world } = defaultBigBang();
   const result = World.worldEqUniverse(world, universe);

client/__tests__/util/stateManager/worldUtil.test.js

@@ -1,62 +0,0 @@
-import {
-  countCategoryValues2D,
-  clearCaches
-} from "../../../src/util/stateManager/worldUtil";
-import * as Dataframe from "../../../src/util/dataframe";
-
-describe("WorldUtil cache management", () => {
-  test("empty", () => {
-    const count = countCategoryValues2D(
-      "a",
-      "b",
-      new Dataframe.Dataframe([0, 0], [])
-    );
-    expect(count).toMatchObject(new Map());
-    expect(count.size).toBe(0);
-  });
-
-  test("simple couts", () => {
-    const df = new Dataframe.Dataframe(
-      [3, 2],
-      [[0, 0, 1], [false, true, false]],
-      null,
-      new Dataframe.KeyIndex(["a", "b"])
-    );
-    const count = countCategoryValues2D("a", "b", df);
-    expect(count).toMatchObject(
-      new Map([
-        [0, new Map([[true, 1], [false, 1]])],
-        [1, new Map([[false, 1]])]
-      ])
-    );
-  });
-
-  test("memo cache clear", () => {
-    clearCaches();
-    const df1 = new Dataframe.Dataframe([0, 0], []);
-    const df2 = new Dataframe.Dataframe(
-      [3, 2],
-      [[0, 0, 1], [false, true, false]],
-      null,
-      new Dataframe.KeyIndex(["a", "b"])
-    );
-
-    const count1 = countCategoryValues2D("a", "b", df1);
-    const count2 = countCategoryValues2D("a", "b", df1);
-    const count3 = countCategoryValues2D("a", "b", df1.clone());
-    const count4 = countCategoryValues2D("a", "b", df2);
-
-    clearCaches();
-    const count10 = countCategoryValues2D("a", "b", df1);
-    const count11 = countCategoryValues2D("a", "b", df2);
-
-    expect(count1).toEqual(count2);
-    expect(count1).toEqual(count3);
-    expect(count1).toEqual(count10);
-    expect(count1).not.toBe(count3);
-    expect(count1).not.toBe(count10);
-
-    expect(count4).toEqual(count11);
-    expect(count4).not.toBe(count11);
-  });
-});

client/__tests__/util/typedCrossfilter/bitArray.test.js

@@ -118,16 +118,16 @@ describe("selectionCount", () => {
     const dim2 = ba.allocDimension();
     expect(dim2).toBeDefined();
 
-    expect(ba.selectionCount()).toEqual(0);
+    expect(ba.selectionCount).toEqual(0);
     ba.selectAll(dim1);
-    expect(ba.selectionCount()).toEqual(0);
+    expect(ba.selectionCount).toEqual(0);
     ba.selectAll(dim2);
-    expect(ba.selectionCount()).toEqual(defaultTestLength);
+    expect(ba.selectionCount).toEqual(defaultTestLength);
 
     for (let i = 0; i < defaultTestLength; i += 1) {
       ba.deselectOne(dim1, i);
-      expect(ba.selectionCount()).toEqual(defaultTestLength - i - 1);
+      expect(ba.selectionCount).toEqual(defaultTestLength - i - 1);
-      expect(ba.selectionCount()).toEqual(ba.countAllOnes());
+      expect(ba.selectionCount).toEqual(ba.countAllOnes());
     }
 
     ba.freeDimension(dim1);

client/__tests__/util/typedCrossfilter/crossfilter.test.js

@@ -1,434 +0,0 @@
-import _ from "lodash";
-import { polygonContains } from "d3";
-
-import Crossfilter from "../../../src/util/typedCrossfilter";
-
-const someData = [
-  {
-    date: "2011-11-14T16:17:54Z",
-    quantity: 2,
-    total: 190,
-    tip: 100,
-    type: "tab",
-    productIDs: ["001"],
-    coords: [0, 0],
-    nonFinite: 0.0
-  },
-  {
-    date: "2011-11-14T16:20:19Z",
-    quantity: 2,
-    total: 190,
-    tip: 100,
-    type: "tab",
-    productIDs: ["001", "005"],
-    coords: [0.4, 0.4],
-    nonFinite: Number.NaN
-  },
-  {
-    date: "2011-11-14T16:28:54Z",
-    quantity: 1,
-    total: 300,
-    tip: 200,
-    type: "visa",
-    productIDs: ["004", "005"],
-    coords: [0.3, 0.1],
-    nonFinite: Number.POSITIVE_INFINITY
-  },
-  {
-    date: "2011-11-14T16:30:43Z",
-    quantity: 2,
-    total: 90,
-    tip: 0,
-    type: "tab",
-    productIDs: ["001", "002"],
-    coords: [0.392, 0.1],
-    nonFinite: Number.NEGATIVE_INFINITY
-  },
-  {
-    date: "2011-11-14T16:48:46Z",
-    quantity: 2,
-    total: 90,
-    tip: 0,
-    type: "tab",
-    productIDs: ["005"],
-    coords: [0.7, 0.0482],
-    nonFinite: 1.0
-  },
-  {
-    date: "2011-11-14T16:53:41Z",
-    quantity: 2,
-    total: 90,
-    tip: 0,
-    type: "tab",
-    productIDs: ["001", "004", "005"],
-    coords: [0.9999, 1.0],
-    nonFinite: Number.NaN
-  },
-  {
-    date: "2011-11-14T16:54:06Z",
-    quantity: 1,
-    total: 100,
-    tip: 0,
-    type: "cash",
-    productIDs: ["001", "002", "003", "004", "005"],
-    coords: [0.384, 0.6938],
-    nonFinite: 99.0
-  },
-  {
-    date: "2011-11-14T16:58:03Z",
-    quantity: 2,
-    total: 90,
-    tip: 0,
-    type: "tab",
-    productIDs: ["001"],
-    coords: [0.4822, 0.482],
-    nonFinite: Number.NaN
-  },
-  {
-    date: "2011-11-14T17:07:21Z",
-    quantity: 2,
-    total: 90,
-    tip: 0,
-    type: "tab",
-    productIDs: ["004", "005"],
-    coords: [0.2234, 0],
-    nonFinite: Number.NaN
-  },
-  {
-    date: "2011-11-14T17:22:59Z",
-    quantity: 2,
-    total: 90,
-    tip: 0,
-    type: "tab",
-    productIDs: ["001", "002", "004", "005"],
-    coords: [0.382, 0.38485],
-    nonFinite: -1
-  },
-  {
-    date: "2011-11-14T17:25:45Z",
-    quantity: 2,
-    total: 200,
-    tip: 0,
-    type: "cash",
-    productIDs: ["002"],
-    coords: [0.998, 0.8472],
-    nonFinite: 0.0
-  },
-  {
-    date: "2011-11-14T17:29:52Z",
-    quantity: 1,
-    total: 200,
-    tip: 100,
-    type: "visa",
-    productIDs: ["004"],
-    coords: [0.8273, 0.3384],
-    nonFinite: 0.0
-  }
-];
-
-let payments = null;
-beforeEach(() => {
-  payments = new Crossfilter(someData);
-});
-
-describe("ImmutableTypedCrossfilter", () => {
-  test("create crossfilter", () => {
-    expect(payments).toBeDefined();
-    expect(payments.size()).toEqual(someData.length);
-    expect(payments.all()).toEqual(someData);
-
-    const p = payments
-      .addDimension("quantity", "scalar", (i, d) => d[i].quantity, Int32Array)
-      .select("quantity", { mode: "all" });
-    expect(p).toBeDefined();
-    expect(p.all()).toEqual(someData);
-    expect(p.size()).toEqual(someData.length);
-    expect(p.isElementSelected(0)).toBeTruthy();
-    expect(p.countSelected()).toEqual(someData.length);
-    expect(p.allSelected()).toEqual(someData);
-  });
-
-  test("immutability", () => {
-    /*
-    the following should return a new crossfilter:
-    - addDimension()
-    - delDimension()
-    - select
-    */
-    const p2 = payments.addDimension(
-      "quantity",
-      "scalar",
-      (i, data) => data[i].quantity,
-      Int32Array
-    );
-
-    expect(payments).not.toBe(p2);
-    const p3 = p2.select("quantity", { mode: "all" });
-    expect(p3).not.toBe(p2);
-
-    const p4 = p3.delDimension("quantity");
-    expect(p4).not.toBe(p3);
-  });
-
-  test("select all and none", () => {
-    let p = payments
-      .addDimension("quantity", "scalar", (i, d) => d[i].quantity, Int32Array)
-      .addDimension("tip", "scalar", (i, d) => d[i].tip, Float32Array)
-      .addDimension("total", "scalar", (i, d) => d[i].total, Float32Array)
-      .addDimension("type", "enum", (i, d) => d[i].type);
-    expect(p).toBeDefined();
-
-    /* expect all records to be selected - default init state */
-    expect(p.allSelected()).toEqual(someData);
-    expect(p.countSelected()).toEqual(someData.length);
-    expect(p.allSelectedMask()).toEqual(
-      new Uint8Array(someData.length).fill(1)
-    );
-    expect(p.fillByIsSelected(new Uint8Array(someData.length), 99, 0)).toEqual(
-      new Uint8Array(someData.length).fill(99)
-    );
-    for (let i = 0; i < someData.length; i += 1) {
-      expect(p.isElementSelected(i)).toBeTruthy();
-    }
-
-    /* expect a selectAll on one dimension to change nothing */
-    p = p.select("tip", { mode: "all" });
-    expect(p.allSelected()).toEqual(someData);
-
-    /* ditto */
-    p = p.select("quantity", { mode: "all" });
-    expect(p.allSelected()).toEqual(someData);
-
-    /* select none on one dimension */
-    p = p.select("type", { mode: "none" });
-    expect(p.allSelected()).toEqual([]);
-    expect(p.countSelected()).toEqual(0);
-    expect(p.allSelectedMask()).toEqual(
-      new Uint8Array(someData.length).fill(0)
-    );
-    expect(p.fillByIsSelected(new Uint8Array(someData.length), 99, 0)).toEqual(
-      new Uint8Array(someData.length).fill(0)
-    );
-    for (let i = 0; i < someData.length; i += 1) {
-      expect(p.isElementSelected(i)).toBeFalsy();
-    }
-
-    p = p.select("quantity", { mode: "none" });
-    expect(p.allSelected()).toEqual([]);
-
-    // invert the first none; should have no effect because type is
-    // still not filtered.
-    p = p.select("quantity", { mode: "all" });
-    expect(p.allSelected()).toEqual([]);
-
-    /* select all of type; should select all records */
-    p = p.select("type", { mode: "all" });
-    expect(p.allSelected()).toEqual(someData);
-  });
-
-  describe("scalar dimension", () => {
-    let p;
-    beforeEach(() => {
-      p = payments
-        .addDimension("quantity", "scalar", (i, d) => d[i].quantity, Int32Array)
-        .addDimension("tip", "scalar", (i, d) => d[i].tip, Float32Array)
-        .select("tip", { mode: "all" });
-    });
-
-    /*
-    select modes:  all, none, exact, range
-    */
-    test("all", () => {
-      expect(p.select("quantity", { mode: "all" }).countSelected()).toEqual(
-        someData.length
-      );
-    });
-    test("none", () => {
-      expect(p.select("quantity", { mode: "none" }).countSelected()).toEqual(0);
-    });
-    test.each([[[]], [[2]], [[2, 1]], [[9, 82]], [[0, 1]]])("exact: %p", v =>
-      expect(
-        p.select("quantity", { mode: "exact", values: v }).countSelected()
-      ).toEqual(_.filter(someData, d => v.includes(d.quantity)).length)
-    );
-    test.each([[0, 1], [1, 2], [0, 99], [99, 100000]])("range %p", (lo, hi) =>
-      expect(
-        p.select("quantity", { mode: "range", lo, hi }).countSelected()
-      ).toEqual(
-        _.filter(someData, d => d.quantity >= lo && d.quantity < hi).length
-      )
-    );
-    test("bad mode", () => {
-      expect(() => p.select("type", { mode: "bad mode" })).toThrow(Error);
-    });
-  });
-
-  describe("enum dimension", () => {
-    let p;
-    beforeEach(() => {
-      p = payments.addDimension("type", "enum", (i, d) => d[i].type);
-    });
-
-    test("all", () => {
-      expect(p.select("type", { mode: "all" }).countSelected()).toEqual(
-        someData.length
-      );
-    });
-    test("none", () => {
-      expect(p.select("type", { mode: "none" }).countSelected()).toEqual(0);
-    });
-    test.each([
-      [[]],
-      [["tab"]],
-      [["visa"]],
-      [["visa", "tab"]],
-      [["cash", "tab", "visa"]]
-    ])("exact: %p", v =>
-      expect(
-        p.select("type", { mode: "exact", values: v }).countSelected()
-      ).toEqual(_.filter(someData, d => v.includes(d.type)).length)
-    );
-    test("range", () => {
-      expect(() => p.select("type", { mode: "range", lo: 0, hi: 9 })).toThrow(
-        Error
-      );
-    });
-    test("bad mode", () => {
-      expect(() => p.select("type", { mode: "bad mode" })).toThrow(Error);
-    });
-  });
-
-  describe("spatial dimension", () => {
-    let p;
-    beforeEach(() => {
-      const X = someData.map(r => r.coords[0]);
-      const Y = someData.map(r => r.coords[1]);
-      p = payments.addDimension("coords", "spatial", X, Y);
-    });
-
-    test("all", () => {
-      expect(p.select("coords", { mode: "all" }).countSelected()).toEqual(
-        someData.length
-      );
-    });
-    test("none", () => {
-      expect(p.select("coords", { mode: "none" }).countSelected()).toEqual(0);
-    });
-    test.each([[0, 0, 1, 1], [0, 0, 0.5, 0.5], [0.5, 0.5, 1, 1]])(
-      "within-rect %d %d %d %d",
-      (minX, minY, maxX, maxY) => {
-        expect(
-          p
-            .select("coords", { mode: "within-rect", minX, minY, maxX, maxY })
-            .allSelected()
-        ).toEqual(
-          _.filter(someData, d => {
-            const [x, y] = d.coords;
-            return minX <= x && x < maxX && minY <= y && y < maxY;
-          })
-        );
-      }
-    );
-
-    test.each([
-      [[[0, 0], [0, 1], [1, 1], [1, 0]]],
-      [[[0, 0], [0, 0.5], [0.5, 0.5], [0.5, 0]]]
-    ])("within-polygon %p", polygon => {
-      expect(
-        p.select("coords", { mode: "within-polygon", polygon }).allSelected()
-      ).toEqual(_.filter(someData, d => polygonContains(polygon, d.coords)));
-    });
-  });
-
-  describe("non-finite scalars", () => {
-    let p;
-    beforeEach(() => {
-      p = payments
-        .addDimension("quantity", "scalar", (i, d) => d[i].quantity, Int32Array)
-        .addDimension(
-          "nonFinite",
-          "scalar",
-          (i, d) => d[i].nonFinite,
-          Float32Array
-        )
-        .select("quantity", { mode: "all" });
-    });
-
-    test("all or none", () => {
-      expect(p.select("nonFinite", { mode: "all" }).countSelected()).toEqual(
-        someData.length
-      );
-      expect(p.select("nonFinite", { mode: "none" }).countSelected()).toEqual(
-        0
-      );
-    });
-
-    test("exact", () => {
-      expect(
-        p.select("nonFinite", { mode: "exact", values: [0] }).countSelected()
-      ).toEqual(3);
-      expect(
-        p.select("nonFinite", { mode: "exact", values: [1] }).countSelected()
-      ).toEqual(1);
-      expect(
-        p
-          .select("nonFinite", {
-            mode: "exact",
-            values: [Number.POSITIVE_INFINITY]
-          })
-          .countSelected()
-      ).toEqual(1);
-      expect(
-        p
-          .select("nonFinite", {
-            mode: "exact",
-            values: [Number.NEGATIVE_INFINITY]
-          })
-          .countSelected()
-      ).toEqual(1);
-      expect(
-        p
-          .select("nonFinite", { mode: "exact", values: [Number.NaN] })
-          .countSelected()
-      ).toEqual(4);
-      expect(
-        p
-          .select("nonFinite", {
-            mode: "exact",
-            values: [Number.POSITIVE_INFINITY, 0, 1, 99]
-          })
-          .countSelected()
-      ).toEqual(6);
-    });
-
-    test("range", () => {
-      expect(
-        p
-          .select("nonFinite", {
-            mode: "range",
-            lo: 0,
-            hi: Number.POSITIVE_INFINITY
-          })
-          .countSelected()
-      ).toEqual(5);
-      expect(
-        p
-          .select("nonFinite", {
-            mode: "range",
-            lo: 0,
-            hi: Number.NaN
-          })
-          .countSelected()
-      ).toEqual(6);
-      expect(
-        p
-          .select("nonFinite", {
-            mode: "range",
-            lo: Number.NEGATIVE_INFINITY,
-            hi: Number.POSITIVE_INFINITY
-          })
-          .countSelected()
-      ).toEqual(7);
-    });
-  });
-});

client/__tests__/util/typedCrossfilter/sort.test.js

@@ -1,22 +1,4 @@
-import {
+import { sort, sortIndex } from "../../../src/util/typedCrossfilter/sort";
-  sortArray,
-  sortIndex,
-  lowerBound,
-  upperBound,
-  lowerBoundIndirect,
-  upperBoundIndirect
-} from "../../../src/util/typedCrossfilter/sort";
-
-/*
-Sort tests should keep in mind that there are separate code
-paths for:
-  - small vs. large arrays (insertionsort only)
-  - float-only typed arrays vs. other array types (non-finite handling)
-  - indexed vs. direct sort
-*/
-
-const pInf = Number.POSITIVE_INFINITY;
-const nInf = Number.NEGATIVE_INFINITY;
 
 function fillRange(arr, start = 0) {
   const larr = arr;
@@ -33,223 +15,42 @@ function fillRand(arr) {
   return arr;
 }
 
-describe("sortArray", () => {
+describe("sort", () => {
-  describe("JS vals", () => {
+  [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map(Type =>
-    [
+    test(Type.name, () => {
-      [true, false],
+      expect(sort(Type.from([6, 5, 4, 3, 2, 1, 0]))).toMatchObject(
-      ["a", "b", "0", "1"],
+        Type.from([0, 1, 2, 3, 4, 5, 6])
-      [0, "a", true, null, undefined, 3.1415],
+      );
-      fillRand(new Array(1000)),
+      expect(sort(Type.from([6, 5, 4, 3, 2, 1]))).toMatchObject(
-      ["a", NaN, null, pInf]
+        Type.from([1, 2, 3, 4, 5, 6])
-    ].map((val, idx) =>
+      );
-      test(`JS vals ${idx}`, () => {
-        expect(sortArray(val)).toMatchObject(val.sort());
-      })
-    );
-  });
 
-  describe("finite numbers", () => {
+      const source = fillRand(new Type(1000));
-    [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map(Type =>
+      expect(sort(Type.from(source))).toMatchObject(Type.from(source).sort());
-      test(Type.name, () => {
+    })
-        expect(sortArray(Type.from([6, 5, 4, 3, 2, 1, 0]))).toMatchObject(
+  );
-          Type.from([0, 1, 2, 3, 4, 5, 6])
-        );
-
-        expect(sortArray(Type.from([6, 5, 4, 3, 2, 1]))).toMatchObject(
-          Type.from([1, 2, 3, 4, 5, 6])
-        );
-
-        const source = fillRand(new Type(1000));
-        expect(sortArray(Type.from(source))).toMatchObject(
-          Type.from(source).sort()
-        );
-      })
-    );
-  });
-
-  describe("non-finite numbers", () => {
-    test("inifinity", () => {
-      expect(sortArray(new Float32Array([pInf, nInf, 0, 1, 2]))).toMatchObject(
-        new Float32Array([nInf, 0, 1, 2, pInf])
-      );
-      expect(
-        sortArray(new Float32Array([pInf, nInf, pInf, nInf]))
-      ).toMatchObject(new Float32Array([nInf, nInf, pInf, pInf]));
-      expect(
-        sortArray(new Float32Array([pInf, nInf, pInf, nInf, pInf]))
-      ).toMatchObject(new Float32Array([nInf, nInf, pInf, pInf, pInf]));
-      expect(
-        sortArray(
-          new Float32Array(100).fill(Infinity, 0, 50).fill(-Infinity, 50, 100)
-        )
-      ).toMatchObject(
-        new Float32Array(100).fill(-Infinity, 0, 50).fill(Infinity, 50, 100)
-      );
-    });
-    test("NaN", () => {
-      expect(sortArray(new Float64Array([NaN, 2, 1, 0]))).toMatchObject(
-        new Float64Array([0, 1, 2, NaN])
-      );
-      expect(sortArray(new Float32Array([NaN, 2, 1, 0]))).toMatchObject(
-        new Float32Array([0, 1, 2, NaN])
-      );
-      expect(sortArray(new Float32Array([NaN, 2, NaN, 1, 0]))).toMatchObject(
-        new Float32Array([0, 1, 2, NaN, NaN])
-      );
-      expect(sortArray(new Float32Array([NaN, 2, 1, NaN, 0]))).toMatchObject(
-        new Float32Array([0, 1, 2, NaN, NaN])
-      );
-      expect(
-        sortArray(fillRange(new Float32Array(100)).fill(NaN, 0, 10))
-      ).toMatchObject(fillRange(new Float32Array(100), 10).fill(NaN, 90, 100));
-    });
-
-    test("mixed numbers", () => {
-      expect(
-        sortArray(new Float32Array([NaN, pInf, nInf, NaN, NaN]))
-      ).toMatchObject(new Float32Array([nInf, pInf, NaN, NaN, NaN]));
-      expect(
-        sortArray(new Float32Array([NaN, pInf, nInf, NaN, 1, NaN, 2]))
-      ).toMatchObject(new Float32Array([nInf, 1, 2, pInf, NaN, NaN, NaN]));
-      expect(
-        sortArray(new Float32Array([NaN, pInf, nInf, 0, 1, NaN, 2]))
-      ).toMatchObject(new Float32Array([nInf, 0, 1, 2, pInf, NaN, NaN]));
-      expect(
-        sortArray(
-          fillRange(new Float32Array(100))
-            .fill(NaN, 0, 10)
-            .fill(Infinity, 10, 20)
-        )
-      ).toMatchObject(
-        fillRange(new Float32Array(100), 20)
-          .fill(Infinity, 80, 90)
-          .fill(NaN, 90, 100)
-      );
-    });
-  });
 });
 
 describe("sortIndex", () => {
-  describe("finite numbers", () => {
+  [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map(Type =>
-    [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map(Type =>
+    test(Type.name, () => {
-      test(Type.name, () => {
+      const source1 = Type.from([6, 5, 4, 3, 2, 1, 0]);
-        const source1 = Type.from([6, 5, 4, 3, 2, 1, 0]);
-        const index1 = fillRange(new Uint32Array(source1.length));
-        expect(sortIndex(index1, source1)).toMatchObject(
-          index1.sort((a, b) => source1[a] - source1[b])
-        );
-
-        const source2 = Type.from([6, 5, 4, 3, 2, 1]);
-        const index2 = fillRange(new Uint32Array(source2.length));
-        expect(sortIndex(index2, source2)).toMatchObject(
-          index2.sort((a, b) => source1[a] - source1[b])
-        );
-
-        const source3 = fillRand(new Type(1000));
-        const index3 = fillRange(new Uint32Array(source3.length));
-        expect(sortIndex(index3, source3)).toMatchObject(
-          index3.sort((a, b) => source1[a] - source1[b])
-        );
-      })
-    );
-  });
-
-  describe("non-finite numbers", () => {
-    test("mixed numbers", () => {
-      const source1 = new Float32Array([NaN, pInf, nInf, NaN, 1, NaN, 2]);
       const index1 = fillRange(new Uint32Array(source1.length));
       expect(sortIndex(index1, source1)).toMatchObject(
-        new Uint32Array([2, 4, 6, 1, 0, 3, 5])
+        index1.sort((a, b) => source1[a] - source1[b])
       );
 
-      const source2 = new Float32Array([NaN, pInf, nInf, 0, 1, NaN, 2]);
+      const source2 = Type.from([6, 5, 4, 3, 2, 1]);
       const index2 = fillRange(new Uint32Array(source2.length));
       expect(sortIndex(index2, source2)).toMatchObject(
-        new Uint32Array([2, 3, 4, 6, 1, 0, 5])
+        index2.sort((a, b) => source1[a] - source1[b])
       );
-    });
-  });
-});
 
-describe("lowerBound", () => {
+      const source3 = fillRand(new Type(1000));
-  test("non-float path", () => {
+      const index3 = fillRange(new Uint32Array(source3.length));
-    expect(lowerBound([], 0, 0, 0)).toEqual(0);
+      expect(sortIndex(index3, source3)).toMatchObject(
-
+        index3.sort((a, b) => source1[a] - source1[b])
-    expect(lowerBound([0, 1, 2, 3], -1, 0, 4)).toEqual(0);
+      );
-    expect(lowerBound([0, 1, 2, 3], 0, 0, 4)).toEqual(0);
+    })
-    expect(lowerBound([0, 1, 2, 3], 1, 0, 4)).toEqual(1);
+  );
-    expect(lowerBound([0, 1, 2, 3], 3, 0, 4)).toEqual(3);
-    expect(lowerBound([0, 1, 2, 3], 4, 0, 4)).toEqual(4);
-
-    expect(lowerBound([0, 1, 2, 3, 4], -1, 0, 5)).toEqual(0);
-    expect(lowerBound([0, 1, 2, 3, 4], 0, 0, 5)).toEqual(0);
-    expect(lowerBound([0, 1, 2, 3, 4], 2, 0, 5)).toEqual(2);
-    expect(lowerBound([0, 1, 2, 3, 4], 4, 0, 5)).toEqual(4);
-    expect(lowerBound([0, 1, 2, 3, 4], 5, 0, 5)).toEqual(5);
-
-    expect(lowerBound([0, 2, 4, 6, 8], 5, 0, 5)).toEqual(3);
-    expect(lowerBound([0, 2, 2, 2, 8], 5, 0, 5)).toEqual(4);
-
-    expect(lowerBound([0, 1, 2, 3, 4, 5, 6, 7, 8], 3, 2, 4)).toEqual(3);
-    expect(lowerBound([0, 1, 2, 3, 4, 5, 6, 7, 8], 99, 2, 4)).toEqual(4);
-  });
-
-  test("float path, finites", () => {
-    expect(lowerBound(new Float32Array([0, 1, 2, 3]), 1, 0, 4)).toEqual(1);
-
-    expect(lowerBound(new Float32Array([]), 0, 0, 0)).toEqual(0);
-
-    expect(lowerBound(new Float32Array([0, 1, 2, 3]), -1, 0, 4)).toEqual(0);
-    expect(lowerBound(new Float32Array([0, 1, 2, 3]), 0, 0, 4)).toEqual(0);
-    expect(lowerBound(new Float32Array([0, 1, 2, 3]), 1, 0, 4)).toEqual(1);
-    expect(lowerBound(new Float32Array([0, 1, 2, 3]), 3, 0, 4)).toEqual(3);
-    expect(lowerBound(new Float32Array([0, 1, 2, 3]), 4, 0, 4)).toEqual(4);
-
-    expect(lowerBound(new Float32Array([0, 1, 2, 3, 4]), -1, 0, 5)).toEqual(0);
-    expect(lowerBound(new Float32Array([0, 1, 2, 3, 4]), 0, 0, 5)).toEqual(0);
-    expect(lowerBound(new Float32Array([0, 1, 2, 3, 4]), 2, 0, 5)).toEqual(2);
-    expect(lowerBound(new Float32Array([0, 1, 2, 3, 4]), 4, 0, 5)).toEqual(4);
-    expect(lowerBound(new Float32Array([0, 1, 2, 3, 4]), 5, 0, 5)).toEqual(5);
-
-    expect(lowerBound(new Float32Array([0, 2, 4, 6, 8]), 5, 0, 5)).toEqual(3);
-    expect(lowerBound(new Float32Array([0, 2, 2, 2, 8]), 5, 0, 5)).toEqual(4);
-
-    expect(
-      lowerBound(new Float32Array([0, 1, 2, 3, 4, 5, 6, 7, 8]), 3, 2, 4)
-    ).toEqual(3);
-    expect(
-      lowerBound(new Float32Array([0, 1, 2, 3, 4, 5, 6, 7, 8]), 99, 2, 4)
-    ).toEqual(4);
-  });
-
-  test("float path, non-finite", () => {
-    expect(
-      lowerBound(
-        new Float32Array([-Infinity, 0, 1, Infinity, NaN]),
-        -Infinity,
-        0,
-        5
-      )
-    ).toEqual(0);
-    expect(
-      lowerBound(new Float32Array([-Infinity, 0, 1, Infinity, NaN]), 0, 0, 5)
-    ).toEqual(1);
-    expect(
-      lowerBound(new Float32Array([-Infinity, 0, 1, Infinity, NaN]), 1, 0, 5)
-    ).toEqual(2);
-    expect(
-      lowerBound(new Float32Array([-Infinity, 0, 1, Infinity, NaN]), 2, 0, 5)
-    ).toEqual(3);
-    expect(
-      lowerBound(
-        new Float32Array([-Infinity, 0, 1, Infinity, NaN]),
-        Infinity,
-        0,
-        5
-      )
-    ).toEqual(3);
-    expect(
-      lowerBound(new Float32Array([-Infinity, 0, 1, Infinity, NaN]), NaN, 0, 5)
-    ).toEqual(4);
-  });
 });

client/__tests__/util/typedCrossfilter/typedCrossfilter.test.js

@@ -0,0 +1,462 @@
+// jshint esversion: 6
+import _ from "lodash";
+import crossfilter from "../../../src/util/typedCrossfilter";
+
+const someData = [
+  {
+    date: "2011-11-14T16:17:54Z",
+    quantity: 2,
+    total: 190,
+    tip: 100,
+    type: "tab",
+    productIDs: ["001"]
+  },
+  {
+    date: "2011-11-14T16:20:19Z",
+    quantity: 2,
+    total: 190,
+    tip: 100,
+    type: "tab",
+    productIDs: ["001", "005"]
+  },
+  {
+    date: "2011-11-14T16:28:54Z",
+    quantity: 1,
+    total: 300,
+    tip: 200,
+    type: "visa",
+    productIDs: ["004", "005"]
+  },
+  {
+    date: "2011-11-14T16:30:43Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["001", "002"]
+  },
+  {
+    date: "2011-11-14T16:48:46Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["005"]
+  },
+  {
+    date: "2011-11-14T16:53:41Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["001", "004", "005"]
+  },
+  {
+    date: "2011-11-14T16:54:06Z",
+    quantity: 1,
+    total: 100,
+    tip: 0,
+    type: "cash",
+    productIDs: ["001", "002", "003", "004", "005"]
+  },
+  {
+    date: "2011-11-14T16:58:03Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["001"]
+  },
+  {
+    date: "2011-11-14T17:07:21Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["004", "005"]
+  },
+  {
+    date: "2011-11-14T17:22:59Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["001", "002", "004", "005"]
+  },
+  {
+    date: "2011-11-14T17:25:45Z",
+    quantity: 2,
+    total: 200,
+    tip: 0,
+    type: "cash",
+    productIDs: ["002"]
+  },
+  {
+    date: "2011-11-14T17:29:52Z",
+    quantity: 1,
+    total: 200,
+    tip: 100,
+    type: "visa",
+    productIDs: ["004"]
+  }
+];
+
+function groupReduce(data, valueMap, valueReduce, valueInit) {
+  return _
+    .reduce(
+      data,
+      (acc, value) => {
+        const k = valueMap(value);
+        let r = _.find(acc, o => o.key === k);
+        if (!r) {
+          r = { key: k, value: valueInit() };
+          acc.push(r);
+        }
+        r.value = valueReduce(r.value, value);
+        return acc;
+      },
+      []
+    )
+    .sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
+}
+
+function groupCount(data, map) {
+  return groupReduce(data, map, (p, v) => p + 1, () => 0);
+}
+
+function groupSum(data, map) {
+  return groupReduce(data, map, (p, v) => (p += map(v)), () => 0);
+}
+
+var payments = null;
+beforeEach(() => {
+  payments = crossfilter(someData);
+});
+
+describe("typedCrossfilter", () => {
+  test("alloc and free", () => {
+    expect(payments).toBeDefined();
+    expect(payments.size()).toEqual(someData.length);
+    expect(payments.all()).toEqual(someData);
+
+    const quantity = payments.dimension(r => r.quantity, Int32Array);
+    expect(quantity).toBeDefined();
+    expect(quantity.id()).toBeDefined();
+
+    quantity.dispose();
+    expect(payments.size()).toEqual(someData.length);
+    expect(payments.all()).toEqual(someData);
+  });
+
+  test("filterAll and filterNone", () => {
+    expect(payments).toBeDefined();
+    const quantity = payments.dimension(r => r.quantity, Int32Array);
+    const tip = payments.dimension(r => r.tip, Float32Array);
+    const total = payments.dimension(r => r.total, Float32Array);
+    const type = payments.dimension(r => r.type, "enum");
+
+    expect(quantity).toBeDefined();
+    expect(tip).toBeDefined();
+    expect(total).toBeDefined();
+    expect(type).toBeDefined();
+
+    // initially, all should be filtered
+    expect(payments.allFiltered()).toHaveLength(payments.size());
+    expect(payments.allFiltered()).toEqual(payments.all());
+    expect(payments.countFiltered()).toEqual(someData.length);
+
+    // filterAll
+    tip.filterAll(); // should change nothing
+    expect(payments.allFiltered()).toEqual(payments.all());
+    expect(payments.countFiltered()).toEqual(someData.length);
+
+    // ditto
+    total.filterAll();
+    expect(payments.allFiltered()).toEqual(payments.all());
+    expect(payments.countFiltered()).toEqual(someData.length);
+
+    // filterNone
+    type.filterNone();
+    expect(payments.allFiltered()).toEqual([]);
+    expect(payments.countFiltered()).toEqual(0);
+
+    quantity.filterNone();
+    expect(payments.allFiltered()).toEqual([]);
+    expect(payments.countFiltered()).toEqual(0);
+
+    // invert the first none; should have no effect because type is
+    // still not filtered
+    quantity.filterAll();
+    expect(payments.allFiltered()).toEqual([]);
+    expect(payments.countFiltered()).toEqual(0);
+
+    // filter all of type; should select all
+    type.filterAll();
+    expect(payments.allFiltered()).toEqual(payments.all());
+    expect(payments.countFiltered()).toEqual(payments.size());
+  });
+
+  test("filterExact", () => {
+    expect(payments).toBeDefined();
+    const quantity = payments.dimension(r => r.quantity, Int32Array);
+    const tip = payments.dimension(r => r.tip, Float32Array);
+    const total = payments.dimension(r => r.total, Float32Array);
+    const type = payments.dimension(r => r.type, "enum");
+
+    quantity.filterExact(1);
+    expect(payments.countFiltered()).toEqual(
+      _.countBy(someData, "quantity")[1]
+    );
+    expect(payments.allFiltered()).toEqual(_.filter(someData, { quantity: 1 }));
+
+    tip.filterExact(0);
+    expect(payments.allFiltered()).toEqual(
+      _.filter(someData, { tip: 0, quantity: 1 })
+    );
+
+    type.filterExact("cash");
+    expect(payments.allFiltered()).toEqual(
+      _.filter(someData, { tip: 0, quantity: 1, type: "cash" })
+    );
+  });
+
+  test("filterRange", () => {
+    expect(payments).toBeDefined();
+    const quantity = payments.dimension(r => r.quantity, Int32Array);
+    const tip = payments.dimension(r => r.tip, Float32Array);
+    const total = payments.dimension(r => r.total, Float32Array);
+    const type = payments.dimension(r => r.type, "enum");
+
+    tip.filterRange([0, 91]);
+    expect(payments.allFiltered()).toEqual(
+      _(someData)
+        .filter(r => r.tip >= 0 && r.tip < 91)
+        .value()
+    );
+
+    tip.filterRange([0, 90]);
+    expect(payments.allFiltered()).toEqual(
+      _(someData)
+        .filter(r => r.tip >= 0 && r.tip < 90)
+        .value()
+    );
+
+    tip.filterRange([1, 90]);
+    expect(payments.allFiltered()).toEqual(
+      _(someData)
+        .filter(r => r.tip >= 1 && r.tip < 91)
+        .value()
+    );
+  });
+
+  test("filterEnum", () => {
+    expect(payments).toBeDefined();
+    const quantity = payments.dimension(r => r.quantity, Int32Array);
+    const tip = payments.dimension(r => r.tip, Float32Array);
+    const total = payments.dimension(r => r.total, Float32Array);
+    const type = payments.dimension(r => r.type, "enum");
+
+    type.filterEnum(["tab", "cash"]);
+    expect(payments.allFiltered()).toEqual(
+      _(someData)
+        .filter(r => r.type === "cash" || r.type === "tab")
+        .value()
+    );
+
+    tip.filterEnum([0, 100]);
+    expect(payments.allFiltered()).toEqual(
+      _(someData)
+        .filter(r => r.type === "cash" || r.type === "tab")
+        .filter(r => r.tip === 0 || r.tip === 100)
+        .value()
+    );
+  });
+
+  test("more than 32 dimensions", () => {
+    expect(payments).toBeDefined();
+    const quantity = payments.dimension(r => r.quantity, Int32Array);
+    const tip = payments.dimension(r => r.tip, Float32Array);
+    const total = payments.dimension(r => r.total, Float32Array);
+    const type = payments.dimension(r => r.type, "enum");
+
+    // Create a bunch of fake dimensions to ensure we can handle > 32
+    let dimMap = {};
+    for (let i = 0; i < 65; i++) {
+      dimMap[i] = payments.dimension(r => Math.random(), Float32Array);
+      expect(dimMap[i]).toBeDefined();
+      expect(dimMap[i].id()).toBeDefined();
+    }
+
+    // everything should start as selected/filtered
+    expect(payments.countFiltered()).toEqual(someData.length);
+
+    dimMap[0].filterAll();
+    dimMap[64].filterAll();
+    expect(payments.countFiltered()).toEqual(someData.length);
+
+    dimMap[33].filterNone();
+    expect(payments.allFiltered()).toEqual([]);
+
+    dimMap[33].filterAll();
+    expect(payments.allFiltered()).toEqual(someData);
+  });
+
+  test("group, default mapping, default reducer, no filter", () => {
+    expect(payments).toBeDefined();
+
+    var quantity = payments.dimension(r => r.quantity, Int32Array);
+    var tip = payments.dimension(r => r.tip, Int32Array);
+    var type = payments.dimension(r => r.type, "enum");
+    var total = payments.dimension(r => r.total, Int32Array);
+
+    _.each(
+      {
+        tip: tip.group(r => r),
+        type: type.group(),
+        total: total.group(),
+        quantity: quantity.group()
+      },
+      (grp, k) => {
+        const whatWeExpect = groupCount(someData, v => v[k]);
+        expect(grp.all()).toEqual(whatWeExpect);
+        expect(grp.size()).toEqual(whatWeExpect.length);
+        expect(grp.dispose()).toEqual(grp);
+      }
+    );
+  });
+
+  test("group, custom map, default reducer, no filters", () => {
+    expect(payments).toBeDefined();
+
+    // custom mapping in groups only works for scalar types.  Enums do not
+    // currently implement it.
+
+    const tip = payments.dimension(r => r.tip, Int32Array);
+    const totalX10 = payments.dimension(r => r.total * 10, Int32Array);
+    const type = payments.dimension(r => r.type, "enum");
+
+    const paymentsByTip_A = tip.group();
+    const paymentsByTip_B = tip.group(r => 10 * r);
+    const paymentsByType = type.group(); // identity only
+    const paymentsByTotalX10_A = totalX10.group();
+    const paymentsByTotalX10_B = totalX10.group(r => r / 10);
+
+    expect(paymentsByTip_A.all()).toEqual(groupCount(someData, v => v.tip));
+    expect(paymentsByTip_B.all()).toEqual(
+      groupCount(someData, v => 10 * v.tip)
+    );
+    expect(paymentsByType.all()).toEqual(groupCount(someData, v => v.type));
+    expect(paymentsByTotalX10_A.all()).toEqual(
+      groupCount(someData, v => 10 * v.total)
+    );
+    expect(paymentsByTotalX10_B.all()).toEqual(
+      groupCount(someData, v => (10 * v.total) / 10)
+    );
+
+    for (let i of [
+      paymentsByTip_A,
+      paymentsByTip_B,
+      paymentsByType,
+      paymentsByTotalX10_A,
+      paymentsByTotalX10_B,
+      tip,
+      totalX10,
+      type
+    ]) {
+      expect(i.dispose()).toEqual(i);
+    }
+  });
+
+  test("group, default map, custom reducer, no filters", () => {
+    expect(payments).toBeDefined();
+
+    const total = payments.dimension(r => r.total, Float32Array);
+    const type = payments.dimension(r => r.type, "enum");
+
+    const paymentsByTotal = total.group();
+    const paymentsByType = type.group();
+
+    // reduceCount
+    expect(paymentsByTotal.reduceCount()).toEqual(paymentsByTotal);
+    expect(paymentsByTotal.all()).toEqual(groupCount(someData, v => v.total));
+
+    // reduceSum
+    expect(paymentsByTotal.reduceSum(v => v.total)).toEqual(paymentsByTotal);
+    expect(paymentsByTotal.all()).toEqual(groupSum(someData, v => v.total));
+
+    // use custom reducers (my reducers) - count by three, init 1
+    expect(
+      paymentsByTotal.reduce((p, v) => (p += 3), (p, v) => (p -= 3), () => 1)
+    ).toEqual(paymentsByTotal);
+    expect(paymentsByTotal.all()).toEqual(
+      groupReduce(someData, v => v.total, (p, v) => p + 3, () => 1)
+    );
+
+    for (let i of [paymentsByTotal, paymentsByType, type]) {
+      expect(i.dispose()).toEqual(i);
+    }
+  });
+
+  test("group, default map, default reducer, filters", () => {
+    // From the docs:
+    // Note: a grouping intersects the crossfilter's current filters, except for the
+    // associated dimension's filter. Thus, group methods consider only records that
+    // satisfy every filter except this dimension's filter. So, if the crossfilter of
+    // payments is filtered by type and total, then group by total only observes the
+    // filter by type.
+
+    expect(payments).toBeDefined();
+
+    const tip = payments.dimension(r => r.tip, Int32Array);
+    const total = payments.dimension(r => r.total, Int32Array);
+    const type = payments.dimension(r => r.type, "enum");
+
+    const paymentsByTip = tip.group();
+    const paymentsByTotal = total.group();
+    const paymentsByType = type.group();
+
+    // 1. confirm that changing the filter on a dimension does NOT change that
+    // dimensions groups.
+    {
+      tip.filterAll(), total.filterAll(), type.filterAll();
+      let before = _.cloneDeep(paymentsByTip.all());
+      tip.filterExact(0);
+      expect(paymentsByTip.all()).toEqual(before);
+    }
+
+    // 2. confirm that changing a filter on a different dimension DOES change
+    // all other groups.
+    {
+      tip.filterAll(), total.filterAll(), type.filterAll();
+      const before = _.cloneDeep([paymentsByTotal.all(), paymentsByType.all()]);
+      tip.filterExact(0);
+      const after = [paymentsByTotal.all(), paymentsByType.all()];
+      expect(after).not.toEqual(before);
+      expect(after).toEqual([
+        groupReduce(
+          someData,
+          v => v.total,
+          (p, v) => (v.tip !== 0 ? p : p + 1),
+          () => 0
+        ),
+        groupReduce(
+          someData,
+          v => v.type,
+          (p, v) => (v.tip !== 0 ? p : p + 1),
+          () => 0
+        )
+      ]);
+    }
+
+    for (let i of [
+      paymentsByTip,
+      paymentsByTotal,
+      paymentsByType,
+      tip,
+      total,
+      type
+    ]) {
+      expect(i.dispose()).toEqual(i);
+    }
+  });
+});

client/__tests__/util/typedCrossfilter/util.test.js

@@ -1,8 +1,11 @@
 import {
+  fillRange,
   sliceByIndex,
-  makeSortIndex
+  makeSortIndex,
+  lowerBound,
+  lowerBoundIndirect,
+  upperBoundIndirect
 } from "../../../src/util/typedCrossfilter/util";
-import { rangeFill as fillRange } from "../../../src/util/range";
 
 describe("fillRange", () => {
   test("Array", () => {

client/configuration/babel/babel.dev.js

@@ -9,8 +9,6 @@ module.exports = {
     "@babel/plugin-proposal-function-bind",
     "@babel/plugin-proposal-class-properties",
     ["@babel/plugin-proposal-decorators", { legacy: true }],
-    "@babel/plugin-proposal-export-namespace-from",
+    "@babel/plugin-proposal-export-namespace-from"
-    "@babel/plugin-proposal-optional-chaining",
-    "@babel/plugin-proposal-nullish-coalescing-operator"
   ]
 };

client/configuration/babel/babel.prod.js

@@ -10,8 +10,6 @@ module.exports = {
     ["@babel/plugin-proposal-decorators", { legacy: true }],
     "@babel/plugin-proposal-export-namespace-from",
     "@babel/plugin-transform-react-constant-elements",
-    "@babel/plugin-transform-runtime",
+    "@babel/plugin-transform-runtime"
-    "@babel/plugin-proposal-optional-chaining",
-    "@babel/plugin-proposal-nullish-coalescing-operator"
   ]
 };

client/configuration/webpack/webpack.config.dev.js

@@ -77,9 +77,6 @@ module.exports = {
       template: path.resolve("index.html"),
       favicon: path.resolve("favicon.png")
     }),
-    new webpack.NoEmitOnErrorsPlugin(),
+    new webpack.NoEmitOnErrorsPlugin()
-    new webpack.DefinePlugin({
-      __REACT_DEVTOOLS_GLOBAL_HOOK__: "({ isDisabled: true })"
-    })
   ]
 };

client/configuration/webpack/webpack.config.prod.js

@@ -10,7 +10,7 @@ const nodeModules = path.resolve("node_modules");
 
 const babelOptions = require("../babel/babel.prod");
 
-const publicPath = "";
+const publicPath = "/";
 
 module.exports = {
   mode: "production",

client/index_template.html

@@ -1,42 +1,35 @@
 <!DOCTYPE html>
 <html lang="en">
-<head>
+  <head>
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1">
     <title>cellxgene</title>
     <link href="https://fonts.googleapis.com/css?family=Roboto+Condensed:400,400i,700" rel="stylesheet">
     <style>
-        html, body, p, h1, h2, h3, h4, h5, h6, span, button, input, label, text, div {
+      html, body, p, h1, h2, h3, h4, h5, h6, span, button, input, label, text, div {
-            font-family: 'Roboto Condensed', 'Helvetica Neue', 'Helvetica', 'Arial', sans-serif;
+        font-family: 'Roboto Condensed','Helvetica Neue','Helvetica','Arial',sans-serif;
-            font-size: 14px;
+        font-size: 14px;
-        }
+      }
+      body {
+        margin: 0;
+        padding: 0;
+      }
 
-        body {
+      * {
-            margin: 0;
+        box-sizing: border-box;
-            padding: 0;
+      }
-        }
-
-        * {
-            box-sizing: border-box;
-        }
     </style>
-
+  </head>
-</head>
+  <body>
-<body>
 <script type="text/javascript">
-  window.CELLXGENE = {};
+window.CELLXGENE = {};
-  window.CELLXGENE.API = {
+window.CELLXGENE.API = {
-    prefix: window.location.href + "api/",
+  prefix: "{{ prefix | safe }}",
-    version: "v0.2/"
+  version: "v0.2/"
-  };
+};
 </script>
-<noscript>If you're seeing this message, that means <strong>JavaScript has been disabled on your browser</strong>,
+    <noscript>If you're seeing this message, that means <strong>JavaScript has been disabled on your browser</strong>, please <strong>enable JS</strong> to make this app work.</noscript>
-    please <strong>enable JS</strong> to make this app work.
-</noscript>
 
-<div id="root"></div>
+    <div id="root"></div>
-{% for script in SCRIPTS %}
+  </body>
-<script type="text/javascript" src="{{script | safe}}"></script>
-{% endfor %}
-</body>
 </html>

client/package.json

@@ -1,21 +1,16 @@
 {
   "name": "cellxgene",
-  "version": "0.9.1",
+  "version": "0.2.4",
   "license": "MIT",
   "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
   "repository": "https://github.com/chanzuckerberg/cellxgene",
   "scripts": {
-    "backend-dev": "python3.6 -m venv cellxgene && source cellxgene/bin/activate && yes | pip uninstall cellxgene || true && pip install -e .. && cellxgene launch ",
     "build": "npm run clean && webpack --config configuration/webpack/webpack.config.prod.js",
-    "clean": "rimraf build",
     "dev": "npm run clean && webpack --config configuration/webpack/webpack.config.dev.js",
-    "e2e": "jest --verbose false --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
+    "clean": "rimraf build",
-    "lint": "eslint src",
-    "smoke-test": "start-server-and-test start-server-for-test :5000 e2e",
     "start": "node server/development.js",
-    "start-server-for-test": "cellxgene launch -p 5000 ../example-dataset/pbmc3k.h5ad",
+    "lint": "eslint src",
-    "test": "jest",
+    "test": "jest"
-    "unit-test": "jest --testPathIgnorePatterns e2e"
   },
   "engineStrict": true,
   "engines": {
@@ -24,25 +19,23 @@
   "eslintConfig": {
     "extends": "./configuration/eslint/eslint.js"
   },
-  "eslintIgnore": [
+  "nyc": {
-    "src/util/stateManager/matrix_generated.js"
+    "sourceMap": false,
-  ],
+    "instrument": false
+  },
   "resolutions": {
     "eslint-scope": "3.7.1"
   },
   "dependencies": {
-    "@blueprintjs/core": "^3.15.0",
+    "@blueprintjs/core": "^3.8.0",
     "@blueprintjs/icons": "^3.3.0",
-    "@blueprintjs/select": "^3.8.0",
+    "@blueprintjs/select": "^3.2.1",
     "canvas-fit": "^1.5.0",
     "d3": "^4.10.0",
     "d3-scale-chromatic": "^1.3.0",
-    "flatbuffers": "^1.10.2",
     "font-color-contrast": "^1.0.3",
-    "fuzzysort": "^1.1.4",
     "gl-mat4": "^1.1.4",
     "gl-matrix": "^2.7.1",
-    "is-number": "^7.0.0",
     "key-pressed": "0.0.1",
     "lodash": "^4.17.4",
     "memoize-one": "^4.0.0",
@@ -70,8 +63,6 @@
     "@babel/plugin-proposal-decorators": "^7.0.0",
     "@babel/plugin-proposal-export-namespace-from": "^7.0.0",
     "@babel/plugin-proposal-function-bind": "^7.0.0",
-    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.2.0",
-    "@babel/plugin-proposal-optional-chaining": "^7.2.0",
     "@babel/plugin-transform-react-constant-elements": "^7.0.0",
     "@babel/plugin-transform-runtime": "^7.1.0",
     "@babel/preset-env": "^7.1.5",
@@ -82,32 +73,31 @@
     "babel-eslint": "^10.0.1",
     "babel-jest": "^23.6.0",
     "babel-loader": "^8.0.0",
+    "babel-plugin-istanbul": "^5.1.0",
     "babel-preset-modern-browsers": "^12.0.0",
-    "chalk": "^2.4.2",
+    "chalk": "^2.4.1",
-    "connect-history-api-fallback": "^1.6.0",
+    "connect-history-api-fallback": "^1.3.0",
     "copy-webpack-plugin": "^4.6.0",
     "css-loader": "^1.0.1",
-    "eslint": "^5.13.0",
+    "eslint": "^5.8.0",
     "eslint-config-airbnb": "^17.1.0",
-    "eslint-config-prettier": "^4.0.0",
+    "eslint-config-prettier": "^3.1.0",
-    "eslint-loader": "^2.1.2",
+    "eslint-loader": "^2.1.1",
     "eslint-plugin-filenames": "^1.3.2",
-    "eslint-plugin-import": "^2.16.0",
+    "eslint-plugin-import": "^2.14.0",
-    "eslint-plugin-jest": "^22.2.2",
+    "eslint-plugin-jest": "^21.27.2",
-    "eslint-plugin-jsx-a11y": "^6.2.1",
+    "eslint-plugin-jsx-a11y": "^6.1.1",
-    "eslint-plugin-react": "^7.12.4",
+    "eslint-plugin-react": "^7.11.1",
     "express": "^4.14.0",
     "file-loader": "^2.0.0",
     "html-webpack-inline-source-plugin": "0.0.10",
     "html-webpack-plugin": "^3.2.0",
-    "jest": "^24.1.0",
+    "jest": "^23.5.0",
-    "jest-puppeteer": "^4.1.0",
     "json-loader": "^0.5.4",
     "mini-css-extract-plugin": "^0.4.1",
-    "puppeteer": "^1.15.0",
+    "nyc": "^13.0.1",
-    "rimraf": "^2.6.3",
+    "rimraf": "^2.5.4",
     "serve-favicon": "^2.3.0",
-    "start-server-and-test": "^1.7.11",
     "style-loader": "^0.23.1",
     "sw-precache-webpack-plugin": "^0.11.5",
     "url-loader": "^1.1.0",
@@ -119,10 +109,7 @@
     "testMatch": [
       "**/__tests__/**/?(*.)(spec|test).js?(x)"
     ],
-    "testURL": "http://localhost/",
+    "testURL": "http://localhost/"
-    "setupFiles": [
-      "./__tests__/setupMissingGlobals.js"
-    ]
   },
   "babel": {
     "env": {
@@ -142,9 +129,7 @@
           ],
           "@babel/plugin-proposal-export-namespace-from",
           "@babel/plugin-transform-react-constant-elements",
-          "@babel/plugin-transform-runtime",
+          "@babel/plugin-transform-runtime"
-          "@babel/plugin-proposal-optional-chaining",
-          "@babel/plugin-proposal-nullish-coalescing-operator"
         ]
       }
     }

client/src/actions/index.js

@@ -1,11 +1,11 @@
 // jshint esversion: 6
 import _ from "lodash";
 import * as globals from "../globals";
-import { Universe } from "../util/stateManager";
+import { Universe, kvCache } from "../util/stateManager";
 import {
   catchErrorsWrap,
   doJsonRequest,
-  doBinaryRequest,
+  rangeEncodeIndices,
   dispatchNetworkErrorMessageToUser
 } from "../util/actionHelpers";
 
@@ -13,33 +13,29 @@ import {
 Bootstrap application with the initial data loading.
   * /config - application configuration
   * /schema - schema of dataframe
-  * /annotations - all metadata annotation
+  * /annotations/obs - all metadata annotation
-  * /layout - all default layout
 */
 const doInitialDataLoad = () =>
   catchErrorsWrap(async dispatch => {
     dispatch({ type: "initial data load start" });
 
     try {
-      const requestJson = _(["config", "schema"])
+      const requests = _([
-        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
+        "config",
-        .map(url => doJsonRequest(url))
+        "schema",
-        .value();
-      const requestBinary = _([
         "annotations/obs",
-        "annotations/var?annotation-name=name",
+        "annotations/var",
         "layout/obs"
       ])
         .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
-        .map(url => doBinaryRequest(url))
+        .map(url => doJsonRequest(url))
         .value();
-
+      const results = await Promise.all(requests);
-      const results = await Promise.all(_.concat(requestJson, requestBinary));
 
       /* set config defaults */
       const config = { ...globals.configDefaults, ...results[0].config };
       const [, schema, obsAnno, varAnno, obsLayout] = [...results];
-      const universe = Universe.createUniverseFromResponse(
+      const universe = Universe.createUniverseFromRestV02Response(
         config,
         schema,
         obsAnno,
@@ -65,7 +61,7 @@ Set the view (world) to current selection.   Placeholder for an async action
 which also does re-layout.
 */
 const regraph = () => (dispatch, getState) => {
-  const { universe, world, crossfilter } = getState();
+  const { universe, world, crossfilter } = getState().controls;
   dispatch({
     type: "set World to current selection",
     universe,
@@ -91,47 +87,15 @@ needs expression data.
 Transparently utilizes cached data if it is already present.
 */
 async function _doRequestExpressionData(dispatch, getState, genes) {
-  /* helper for this function only */
-  const fetchData = async geneNames => {
-    const res = await fetch(
-      `${globals.API.prefix}${globals.API.version}data/var`,
-      {
-        method: "PUT",
-        body: JSON.stringify({
-          filter: {
-            var: {
-              annotation_value: [{ name: "name", values: geneNames }]
-            }
-          }
-        }),
-        headers: new Headers({
-          accept: "application/octet-stream",
-          "Content-Type": "application/json"
-        })
-      }
-    );
-
-    if (
-      !res.ok ||
-      res.headers.get("Content-Type") !== "application/octet-stream"
-    ) {
-      // WILL throw
-      return dispatchExpressionErrors(dispatch, res);
-    }
-
-    const data = await res.arrayBuffer();
-    return Universe.convertDataFBStoObject(universe, data);
-  };
-
   const state = getState();
-  const { universe } = state;
+  const { universe } = state.controls;
   /* preload data already in cache */
   let expressionData = _.transform(
     genes,
     (expData, g) => {
-      const data = universe.varData.col(g);
+      const data = kvCache.get(universe.varDataCache, g);
       if (data) {
-        expData[g] = data.asArray();
+        expData[g] = data;
       }
     },
     {}
@@ -144,10 +108,35 @@ async function _doRequestExpressionData(dispatch, getState, genes) {
   /* Fetch data for any genes not in cache */
   if (genesToFetch.length) {
     try {
-      const newExpressionData = await fetchData(genesToFetch);
+      // XXX: TODO - this could be using /data/var rather than /data/obs,
+      // as that would simplify the transformation in convertExpressionRESTv02ToObject
+      const res = await fetch(
+        `${globals.API.prefix}${globals.API.version}data/obs`,
+        {
+          method: "PUT",
+          body: JSON.stringify({
+            filter: {
+              var: {
+                annotation_value: [{ name: "name", values: genesToFetch }]
+              }
+            }
+          }),
+          headers: new Headers({
+            accept: "application/json",
+            "Content-Type": "application/json"
+          })
+        }
+      );
+
+      if (!res.ok || res.headers.get("Content-Type") !== "application/json") {
+        // WILL throw
+        return dispatchExpressionErrors(dispatch, res);
+      }
+
+      const data = await res.json();
       expressionData = {
         ...expressionData,
-        ...newExpressionData
+        ...Universe.convertExpressionRESTv02ToObject(universe, data)
       };
     } catch (error) {
       dispatch({ type: "expression load error", error });
@@ -164,12 +153,12 @@ function requestSingleGeneExpressionCountsForColoringPOST(gene) {
     dispatch({ type: "get single gene expression for coloring started" });
     try {
       await _doRequestExpressionData(dispatch, getState, [gene]);
-      const { world } = getState();
+      const { world } = getState().controls;
       dispatch({
         type: "color by expression",
         gene,
         data: {
-          [gene]: world.varData.col(gene).asArray()
+          [gene]: kvCache.get(world.varDataCache, gene)
         }
       });
     } catch (error) {
@@ -185,14 +174,14 @@ const requestUserDefinedGene = gene => async (dispatch, getState) => {
   dispatch({ type: "request user defined gene started" });
   try {
     await await _doRequestExpressionData(dispatch, getState, [gene]);
-    const { world } = getState();
+    const { world } = getState().controls;
 
     /* then send the success case action through */
     return dispatch({
       type: "request user defined gene success",
       data: {
         genes: [gene],
-        expression: world.varData.col(gene).asArray()
+        expression: kvCache.get(world.varDataCache, gene)
       }
     });
   } catch (error) {
@@ -240,19 +229,13 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
     2. get expression data for each
     */
     const state = getState();
-    const { universe } = state;
+    const { universe } = state.controls;
-
+    const set1ByIndex = rangeEncodeIndices(
-    // Legal values are null, Array or TypedArray.  Null is initial state.
+      _.map(set1, s => universe.obsNameToIndexMap[s])
-    if (!set1) set1 = [];
+    );
-    if (!set2) set2 = [];
+    const set2ByIndex = rangeEncodeIndices(
-
+      _.map(set2, s => universe.obsNameToIndexMap[s])
-    // These lines ensure that we convert any TypedArray to an Array.
+    );
-    // This is necessary because JSON.stringify() does some very strange
-    // things with TypedArrays (they are marshalled to JSON objects, rather
-    // than being marshalled as a JSON array).
-    set1 = Array.isArray(set1) ? set1 : Array.from(set1);
-    set2 = Array.isArray(set2) ? set2 : Array.from(set2);
-
     const res = await fetch(
       `${globals.API.prefix}${globals.API.version}diffexp/obs`,
       {
@@ -264,8 +247,8 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
         body: JSON.stringify({
           mode: "topN",
           count: num_genes,
-          set1: { filter: { obs: { index: set1 } } },
+          set1: { filter: { obs: { index: set1ByIndex } } },
-          set2: { filter: { obs: { index: set2 } } }
+          set2: { filter: { obs: { index: set2ByIndex } } }
         })
       }
     );
@@ -276,9 +259,7 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
 
     const data = await res.json();
     // result is [ [varIdx, ...], ... ]
-    const topNGenes = _.map(data, r =>
+    const topNGenes = _.map(data, r => universe.varAnnotations[r[0]].name);
-      universe.varAnnotations.at(r[0], "name")
-    );
 
     /*
     Kick off secondary action to fetch all of the expression data for the
@@ -300,11 +281,8 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
 };
 
 const resetInterface = () => (dispatch, getState) => {
-  const { universe } = getState();
+  const { universe } = getState().controls;
 
-  dispatch({
-    type: "user reset start"
-  });
   dispatch({
     type: "clear all user defined genes"
   });
@@ -324,9 +302,6 @@ const resetInterface = () => (dispatch, getState) => {
   dispatch({
     type: "increment graph render counter"
   });
-  dispatch({
-    type: "user reset end"
-  });
 };
 
 export default {

client/src/components/brushableHistogram/index.js

@@ -5,65 +5,69 @@ https://bl.ocks.org/SpaceActuary/2f004899ea1b2bd78d6f1dbb2febf771
 */
 // jshint esversion: 6
 import React from "react";
+import _ from "lodash";
 import { Button, ButtonGroup, Tooltip } from "@blueprintjs/core";
 import { connect } from "react-redux";
 import * as d3 from "d3";
 import memoize from "memoize-one";
+import { kvCache } from "../../util/stateManager";
 import * as globals from "../../globals";
 import actions from "../../actions";
-import { makeContinuousDimensionName } from "../../util/nameCreators";
 
 @connect(state => ({
-  world: state.world,
+  world: state.controls.world,
   scatterplotXXaccessor: state.controls.scatterplotXXaccessor,
   scatterplotYYaccessor: state.controls.scatterplotYYaccessor,
-  continuousSelection: state.continuousSelection,
+  crossfilter: state.controls.crossfilter,
   differential: state.differential,
-  colorAccessor: state.colors.colorAccessor
+  initializeRanges: _.get(state.controls.world, "summary.obs"),
+  colorAccessor: state.controls.colorAccessor,
+  colorScale: state.controls.colorScale,
+  obsAnnotations: _.get(state.controls.world, "obsAnnotations", null)
 }))
 class HistogramBrush extends React.Component {
-  static getColumn(world, field, clipped = true) {
+  calcHistogramCache = memoize((obsAnnotations, field, ranges) => {
-    /*
+    const { world } = this.props;
-    Return the underlying Dataframe column for our field.   By default, 
-    returns the clipped column.   If clipped===false, will return the 
-    unclipped column.
-    */
-    const obsAnnotations = clipped
-      ? world.obsAnnotations
-      : world.unclipped.obsAnnotations;
-    const varData = clipped ? world.varData : world.unclipped.varData;
-    if (obsAnnotations.hasCol(field)) {
-      return obsAnnotations.col(field);
-    }
-    return varData.col(field);
-  }
-
-  calcHistogramCache = memoize((col, field) => {
-    /*
-     recalculate expensive stuff, notably bins, summaries, etc.
-    */
     const histogramCache = {};
-    const values = col.asArray();
-    const summary = col.summarize();
-    const { min: domainMin, max: domainMax } = summary;
-    histogramCache.x = d3
-      .scaleLinear()
-      .domain([domainMin, domainMax])
-      .range([0, this.width - this.marginRight]);
 
-    histogramCache.bins = d3
-      .histogram()
-      .domain(histogramCache.x.domain())
-      .thresholds(40)(values);
-
-    const yMax = histogramCache.bins
-      .map(b => b.length)
-      .reduce((a, b) => Math.max(a, b));
     histogramCache.y = d3
       .scaleLinear()
-      .domain([0, yMax])
       .range([this.height - this.marginBottom, 0]);
 
+    if (obsAnnotations[0][field] !== undefined) {
+      // recalculate expensive stuff
+      const allValuesForContinuousFieldAsArray = _.map(obsAnnotations, field);
+
+      histogramCache.x = d3
+        .scaleLinear()
+        .domain([ranges.min, ranges.max])
+        .range([0, this.width]);
+
+      histogramCache.bins = d3
+        .histogram()
+        .domain(histogramCache.x.domain())
+        .thresholds(40)(allValuesForContinuousFieldAsArray);
+
+      histogramCache.numValues = allValuesForContinuousFieldAsArray.length;
+    } else if (kvCache.get(world.varDataCache, field)) {
+      /* it's not in observations, so it's a gene, but let's check to make sure */
+      const varValues = kvCache.get(world.varDataCache, field);
+
+      histogramCache.x = d3
+        .scaleLinear()
+        .domain(
+          d3.extent(varValues)
+        ) /* replace this if we have ranges for genes back from server like we do for annotations on cells */
+        .range([0, this.width]);
+
+      histogramCache.bins = d3
+        .histogram()
+        .domain(histogramCache.x.domain())
+        .thresholds(40)(varValues);
+
+      histogramCache.numValues = varValues.length;
+    }
+
     return histogramCache;
   });
 
@@ -72,98 +76,32 @@ class HistogramBrush extends React.Component {
 
     this.width = 340;
     this.height = 100;
-    this.marginBottom = 20; // space for X axis & labels
+    this.marginBottom = 20;
-    this.marginRight = 40; // space for Y axis & labels
   }
 
   componentDidMount() {
     const { field } = this.props;
-    const { x, y, bins, svgRef } = this._histogram;
+    const { x, y, bins, numValues, svgRef } = this._histogram;
 
-    this.renderAxesBrushBins(x, y, bins, svgRef, field);
+    this.renderAxesBrushBins(x, y, bins, numValues, svgRef, field);
   }
 
   componentDidUpdate(prevProps) {
-    const { field, world, continuousSelection } = this.props;
+    const { field, obsAnnotations } = this.props;
-    const { x, y, bins, svgRef } = this._histogram;
+    const { x, y, bins, numValues, svgRef } = this._histogram;
-    let { brushXselection, brushX } = this.state;
-    let forceBrushUpdate = false;
 
-    /*
+    if (obsAnnotations !== prevProps.obsAnnotations) {
-    Update our axis if the underlying dataframe column has changed
+      this.renderAxesBrushBins(x, y, bins, numValues, svgRef, field);
-    */
-    const dfColumn = HistogramBrush.getColumn(world, field);
-    const oldDfColumn = HistogramBrush.getColumn(
-      prevProps.world,
-      prevProps.field
-    );
-    if (dfColumn !== oldDfColumn) {
-      ({ brushXselection, brushX } = this.renderAxesBrushBins(
-        x,
-        y,
-        bins,
-        svgRef,
-        field
-      ));
-      forceBrushUpdate = true;
-    }
-
-    /*
-    if the selection has changed, ensure that the brush correctly reflects
-    the underlying selection.
-    */
-    if (
-      forceBrushUpdate ||
-      continuousSelection !== prevProps.continuousSelection
-    ) {
-      const { isObs, isUserDefined, isDiffExp } = this.props;
-      const myName = makeContinuousDimensionName(
-        { isObs, isUserDefined, isDiffExp },
-        field
-      );
-      const range = continuousSelection[myName];
-      if (brushXselection) {
-        const selection = d3.brushSelection(brushXselection.node());
-        if (!range && selection) {
-          /* no active selection - clear brush */
-          brushXselection.call(brushX.move, null);
-        } else if (range && !selection) {
-          /* there is an active selection, but no brush - set the brush */
-          const x0 = x(range[0]);
-          const x1 = x(range[1]);
-          brushXselection.call(brushX.move, [x0, x1]);
-        } else if (range && selection) {
-          /* there is an active selection and a brush - make sure they match */
-          const moveDeltaThreshold = 1;
-          const x0 = x(range[0]);
-          const x1 = x(range[1]);
-          const dX0 = Math.abs(x0 - selection[0]);
-          const dX1 = Math.abs(x1 - selection[1]);
-          /*
-          only update the brush if it is grossly incorrect,
-          as defined by the moveDeltaThreshold
-          */
-          if (dX0 > moveDeltaThreshold || dX1 > moveDeltaThreshold) {
-            brushXselection.call(brushX.move, [x0, x1]);
-          }
-        }
-      }
     }
   }
 
-  onBrush(selection, x, eventType) {
+  onBrush(selection, x) {
-    const type = `continuous metadata histogram ${eventType}`;
     return () => {
       const { dispatch, field, isObs, isUserDefined, isDiffExp } = this.props;
 
-      // ignore programmatically generated events
-      if (!d3.event.sourceEvent) return;
-      // ignore cascading events, which are programmatically generated
-      if (d3.event.sourceEvent.sourceEvent) return;
-
       if (d3.event.selection) {
         dispatch({
-          type,
+          type: "continuous metadata histogram brush",
           selection: field,
           continuousNamespace: {
             isObs,
@@ -174,7 +112,7 @@ class HistogramBrush extends React.Component {
         });
       } else {
         dispatch({
-          type,
+          type: "continuous metadata histogram brush",
           selection: field,
           continuousNamespace: {
             isObs,
@@ -187,91 +125,41 @@ class HistogramBrush extends React.Component {
     };
   }
 
-  onBrushEnd(selection, x) {
-    return () => {
-      const { dispatch, field, isObs, isUserDefined, isDiffExp } = this.props;
-      const minAllowedBrushSize = 10;
-      const smallAmountToAvoidInfiniteLoop = 0.1;
-
-      // ignore programmatically generated events
-      if (!d3.event.sourceEvent) return;
-      // ignore cascading events, which are programmatically generated
-      if (d3.event.sourceEvent.sourceEvent) return;
-
-      if (d3.event.selection) {
-        let _range;
-
-        if (
-          d3.event.selection[1] - d3.event.selection[0] >
-          minAllowedBrushSize
-        ) {
-          _range = [x(d3.event.selection[0]), x(d3.event.selection[1])];
-        } else {
-          /* the user selected range is too small and will be hidden #587, so take control of it procedurally */
-          /* https://stackoverflow.com/questions/12354729/d3-js-limit-size-of-brush */
-
-          const procedurallyResizedBrushWidth =
-            d3.event.selection[0] +
-            minAllowedBrushSize +
-            smallAmountToAvoidInfiniteLoop; //
-
-          _range = [x(d3.event.selection[0]), x(procedurallyResizedBrushWidth)];
-        }
-
-        dispatch({
-          type: "continuous metadata histogram end",
-          selection: field,
-          continuousNamespace: {
-            isObs,
-            isUserDefined,
-            isDiffExp
-          },
-          range: _range
-        });
-      } else {
-        dispatch({
-          type: "continuous metadata histogram cancel",
-          selection: field,
-          continuousNamespace: {
-            isObs,
-            isUserDefined,
-            isDiffExp
-          }
-        });
-      }
-    };
-  }
-
   drawHistogram(svgRef) {
-    const { field, world } = this.props;
+    const { obsAnnotations, field, ranges } = this.props;
-    const col = HistogramBrush.getColumn(world, field);
+    const histogramCache = this.calcHistogramCache(
-    const histogramCache = this.calcHistogramCache(col, field);
+      obsAnnotations,
-    const { x, y, bins } = histogramCache;
+      field,
-    this._histogram = { x, y, bins, svgRef };
+      ranges
+    );
+
+    const { x, y, bins, numValues } = histogramCache;
+
+    this._histogram = { x, y, bins, numValues, svgRef };
   }
 
   handleColorAction() {
-    const { dispatch, field, world, ranges } = this.props;
+    const {
+      obsAnnotations,
+      dispatch,
+      field,
+      world,
+      initializeRanges
+    } = this.props;
 
-    if (world.obsAnnotations.hasCol(field)) {
+    if (obsAnnotations[0][field]) {
       dispatch({
         type: "color by continuous metadata",
         colorAccessor: field,
-        rangeForColorAccessor: ranges
+        rangeMaxForColorAccessor: initializeRanges[field].range.max
       });
-    } else if (world.varData.hasCol(field)) {
+    } else if (kvCache.get(world.varDataCache, field)) {
       dispatch(actions.requestSingleGeneExpressionCountsForColoringPOST(field));
     }
   }
 
   removeHistogram() {
-    const {
+    const { dispatch, field, colorAccessor } = this.props;
-      dispatch,
-      field,
-      colorAccessor,
-      scatterplotXXaccessor,
-      scatterplotYYaccessor
-    } = this.props;
     dispatch({
       type: "clear user defined gene",
       data: field
@@ -281,18 +169,6 @@ class HistogramBrush extends React.Component {
         type: "reset colorscale"
       });
     }
-    if (field === scatterplotXXaccessor) {
-      dispatch({
-        type: "set scatterplot x",
-        data: null
-      });
-    }
-    if (field === scatterplotYYaccessor) {
-      dispatch({
-        type: "set scatterplot y",
-        data: null
-      });
-    }
   }
 
   handleSetGeneAsScatterplotX() {
@@ -315,114 +191,81 @@ class HistogramBrush extends React.Component {
     };
   }
 
-  renderAxesBrushBins(x, y, bins, svgRef, field) {
+  renderAxesBrushBins(x, y, bins, numValues, svgRef, field) {
-    const svg = d3.select(svgRef);
-
     /* Remove everything */
-    svg.selectAll("*").remove();
+    d3.select(svgRef)
+      .selectAll("*")
+      .remove();
 
     /* BINS */
-    svg
+    d3.select(svgRef)
       .insert("g", "*")
       .attr("fill", "#bbb")
       .selectAll("rect")
       .data(bins)
       .enter()
       .append("rect")
+      .attr("class", "bar")
       .attr("x", d => x(d.x0) + 1)
-      .attr("y", d => y(d.length))
+      .attr("y", d => y(d.length / numValues))
       .attr("width", d => Math.abs(x(d.x1) - x(d.x0) - 1))
-      .attr("height", d => y(0) - y(d.length));
+      .attr("height", d => y(0) - y(d.length / numValues));
 
     /* BRUSH */
-    const brushX = d3
+    d3.select(svgRef)
-      .brushX()
-      .extent([[0, 0], [this.width - this.marginRight, this.height]])
-      /*
-      emit start so that the Undoable history can save an undo point
-      upon drag start, and ignore the subsequent intermediate drag events.
-      */
-      .on("start", this.onBrush(field, x.invert, "start").bind(this))
-      .on("brush", this.onBrush(field, x.invert, "brush").bind(this))
-      .on("end", this.onBrushEnd(field, x.invert).bind(this));
-    const brushXselection = d3
-      .select(svgRef)
       .append("g")
       .attr("class", "brush")
-      .attr("data-testid", `${svgRef.dataset.testid}-brush`)
+      .call(
-      .call(brushX);
+        d3
+          .brushX()
+          .on("brush", this.onBrush(field, x.invert).bind(this))
+          .on("end", this.onBrush(field, x.invert).bind(this))
+      );
 
-    /* X AXIS */
+    /* AXIS */
-    svg
+    d3.select(svgRef)
       .append("g")
       .attr("class", "axis axis--x")
       .attr("transform", `translate(0,${this.height - this.marginBottom})`)
       .call(d3.axisBottom(x).ticks(5));
 
-    /* Y AXIS */
+    d3.select(svgRef)
-    svg
+      .selectAll(".axis--x text")
-      .append("g")
+      .style("fill", "rgb(80,80,80)");
-      .attr("class", "axis axis--y")
-      .attr("transform", `translate(${this.width - this.marginRight},0)`)
-      .call(d3.axisRight(y).ticks(3));
 
-    /* axis style */
+    d3.select(svgRef)
-    svg.selectAll(".axis text").style("fill", "rgb(80,80,80)");
+      .selectAll(".axis--x path")
-    svg.selectAll(".axis path").style("stroke", "rgb(230,230,230)");
+      .style("stroke", "rgb(230,230,230)");
-    svg.selectAll(".axis line").style("stroke", "rgb(230,230,230)");
 
-    const newState = { brushX, brushXselection };
+    d3.select(svgRef)
-    this.setState(newState);
+      .selectAll(".axis--x line")
-    return newState;
+      .style("stroke", "rgb(230,230,230)");
   }
 
   render() {
     const {
       field,
-      world,
       colorAccessor,
       isUserDefined,
       isDiffExp,
       logFoldChange,
+      pval,
       pvalAdj,
       scatterplotXXaccessor,
       scatterplotYYaccessor,
       zebra
     } = this.props;
-    const fieldForId = field.replace(/\s/g, "_");
-    const {
-      min: unclippedRangeMin,
-      max: unclippedRangeMax
-    } = HistogramBrush.getColumn(world, field, false).summarize();
-    const unclippedRangeMinColor =
-      world.clipQuantiles.min === 0 ? "#bbb" : globals.blue;
-    const unclippedRangeMaxColor =
-      world.clipQuantiles.max === 1 ? "#bbb" : globals.blue;
 
     return (
       <div
-        id={`histogram_${fieldForId}`}
+        id={`histogram_${field}`}
-        data-testid={`histogram-${field}`}
-        data-testclass={
-          isDiffExp
-            ? "histogram-diffexp"
-            : isUserDefined
-            ? "histogram-user-gene"
-            : "histogram-continuous-metadata"
-        }
         style={{
           padding: globals.leftSidebarSectionPadding,
           backgroundColor: zebra ? globals.lightestGrey : "white"
         }}
       >
-        <div
+        <div style={{ display: "flex", justifyContent: "flex-end" }}>
-          style={{
+          {isDiffExp ? (
-            display: "flex",
-            justifyContent: "flex-end",
-            paddingBottom: "8px"
-          }}
-        >
-          {isDiffExp || isUserDefined ? (
             <span>
               <span
                 style={{ marginRight: 7 }}
@@ -430,7 +273,6 @@ class HistogramBrush extends React.Component {
               />
               <ButtonGroup style={{ marginRight: 7 }}>
                 <Button
-                  data-testid={`plot-x-${field}`}
                   onClick={this.handleSetGeneAsScatterplotX(field).bind(this)}
                   active={scatterplotXXaccessor === field}
                   intent={scatterplotXXaccessor === field ? "primary" : "none"}
@@ -438,7 +280,6 @@ class HistogramBrush extends React.Component {
                   plot x
                 </Button>
                 <Button
-                  data-testid={`plot-y-${field}`}
                   onClick={this.handleSetGeneAsScatterplotY(field).bind(this)}
                   active={scatterplotYYaccessor === field}
                   intent={scatterplotYYaccessor === field ? "primary" : "none"}
@@ -466,8 +307,6 @@ class HistogramBrush extends React.Component {
               onClick={this.handleColorAction.bind(this)}
               active={colorAccessor === field}
               intent={colorAccessor === field ? "primary" : "none"}
-              data-testclass="colorby"
-              data-testid={`colorby-${field}`}
               icon="tint"
             />
           </Tooltip>
@@ -475,9 +314,7 @@ class HistogramBrush extends React.Component {
         <svg
           width={this.width}
           height={this.height}
-          id={`histogram_${fieldForId}_svg`}
+          id={`histogram_${field}_svg`}
-          data-testclass="histogram-plot"
-          data-testid={`histogram-${field}-plot`}
           ref={svgRef => {
             this.drawHistogram(svgRef);
           }}
@@ -485,21 +322,10 @@ class HistogramBrush extends React.Component {
         <div
           style={{
             display: "flex",
-            justifyContent: "space-between"
+            justifyContent: "center"
           }}
         >
-          <span style={{ color: unclippedRangeMinColor }}>
+          <span style={{ fontStyle: "italic" }}>{field}</span>
-            min {unclippedRangeMin.toPrecision(4)}
-          </span>
-          <span
-            data-testclass="brushable-histogram-field-name"
-            style={{ fontStyle: "italic" }}
-          >
-            {field}
-          </span>
-          <span style={{ color: unclippedRangeMaxColor }}>
-            max {unclippedRangeMax.toPrecision(4)}
-          </span>
         </div>
 
         {isDiffExp ? (

client/src/components/categorical/categorical.js

@@ -6,12 +6,12 @@ import * as globals from "../../globals";
 import Category from "./category";
 
 @connect(state => ({
-  categoricalSelection: state.categoricalSelection
+  categoricalSelectionState: state.controls.categoricalSelectionState
 }))
 class Categories extends React.Component {
   render() {
-    const { categoricalSelection } = this.props;
+    const { categoricalSelectionState } = this.props;
-    if (!categoricalSelection) return null;
+    if (!categoricalSelectionState) return null;
 
     return (
       <div
@@ -26,7 +26,7 @@ class Categories extends React.Component {
         >
           Categorical Metadata
         </p>
-        {_.map(categoricalSelection, (catState, catName) => (
+        {_.map(categoricalSelectionState, (catState, catName) => (
           <Category key={catName} metadataField={catName} />
         ))}
       </div>

client/src/components/categorical/category.js

@@ -6,11 +6,11 @@ import { Button, Tooltip } from "@blueprintjs/core";
 
 import * as globals from "../../globals";
 import Value from "./value";
-import sortedCategoryValues from "./util";
+import alphabeticallySortedValues from "./util";
 
 @connect(state => ({
-  colorAccessor: state.colors.colorAccessor,
+  colorAccessor: state.controls.colorAccessor,
-  categoricalSelection: state.categoricalSelection
+  categoricalSelectionState: state.controls.categoricalSelectionState
 }))
 class Category extends React.Component {
   constructor(props) {
@@ -21,43 +21,40 @@ class Category extends React.Component {
     };
   }
 
-  componentDidUpdate(prevProps) {
+  componentDidUpdate() {
-    const { categoricalSelection, metadataField } = this.props;
+    const { categoricalSelectionState, metadataField } = this.props;
-    if (categoricalSelection !== prevProps.categoricalSelection) {
+    const cat = categoricalSelectionState[metadataField];
-      const cat = categoricalSelection[metadataField];
+    const categoryCount = {
-      const categoryCount = {
+      // total number of options in this category
-        // total number of categories in this dimension
+      totalOptionCount: cat.numOptions,
-        totalCatCount: cat.numCategories,
+      // number of selected options in this category
-        // number of selected options in this category
+      selectedOptionCount: _.reduce(
-        selectedCatCount: _.reduce(
+        cat.optionSelected,
-          cat.categorySelected,
+        (res, cond) => (cond ? res + 1 : res),
-          (res, cond) => (cond ? res + 1 : res),
+        0
-          0
+      )
-        )
+    };
-      };
+    if (categoryCount.selectedOptionCount === categoryCount.totalOptionCount) {
-      if (categoryCount.selectedCatCount === categoryCount.totalCatCount) {
+      /* everything is on, so not indeterminate */
-        /* everything is on, so not indeterminate */
+      this.checkbox.indeterminate = false;
-        this.checkbox.indeterminate = false;
+    } else if (categoryCount.selectedOptionCount === 0) {
-        this.setState({ isChecked: true }); // eslint-disable-line react/no-did-update-set-state
+      /* nothing is on, so no */
-      } else if (categoryCount.selectedCatCount === 0) {
+      this.checkbox.indeterminate = false;
-        /* nothing is on, so no */
+    } else if (
-        this.checkbox.indeterminate = false;
+      categoryCount.selectedOptionCount < categoryCount.totalOptionCount
-        this.setState({ isChecked: false }); // eslint-disable-line react/no-did-update-set-state
+    ) {
-      } else if (categoryCount.selectedCatCount < categoryCount.totalCatCount) {
+      /* to be explicit... */
-        /* to be explicit... */
+      this.checkbox.indeterminate = true;
-        this.checkbox.indeterminate = true;
-        this.setState({ isChecked: false });
-      }
     }
   }
 
-  handleColorChange = () => {
+  handleColorChange() {
     const { dispatch, metadataField } = this.props;
     dispatch({
       type: "color by categorical metadata",
       colorAccessor: metadataField
     });
-  };
+  }
 
   toggleAll() {
     const { dispatch, metadataField } = this.props;
@@ -88,16 +85,15 @@ class Category extends React.Component {
   }
 
   renderCategoryItems() {
-    const { categoricalSelection, metadataField } = this.props;
+    const { categoricalSelectionState, metadataField } = this.props;
 
-    const cat = categoricalSelection[metadataField];
+    const cat = categoricalSelectionState[metadataField];
-    const optTuples = sortedCategoryValues([...cat.categoryIndices]);
+    const optTuples = alphabeticallySortedValues([...cat.optionIndex]);
     return _.map(optTuples, (tuple, i) => (
       <Value
-        optTuples={optTuples}
         key={tuple[1]}
         metadataField={metadataField}
-        categoryIndex={tuple[1]}
+        optionIndex={tuple[1]}
         i={i}
       />
     ));
@@ -105,15 +101,17 @@ class Category extends React.Component {
 
   render() {
     const { isExpanded, isChecked } = this.state;
-    const { metadataField, colorAccessor, categoricalSelection } = this.props;
+    const {
-    const { isTruncated } = categoricalSelection[metadataField];
+      metadataField,
+      colorAccessor,
+      categoricalSelectionState
+    } = this.props;
+    const { isTruncated } = categoricalSelectionState[metadataField];
     return (
       <div
         style={{
           maxWidth: globals.maxControlsWidth
         }}
-        data-testclass="category"
-        data-testid={`category-${metadataField}`}
       >
         <div
           style={{
@@ -131,8 +129,6 @@ class Category extends React.Component {
           >
             <label className="bp3-control bp3-checkbox">
               <input
-                data-testclass="category-select"
-                data-testid={`category-select-${metadataField}`}
                 onChange={this.handleToggleAllClick.bind(this)}
                 ref={el => {
                   this.checkbox = el;
@@ -146,7 +142,6 @@ class Category extends React.Component {
             </label>
 
             <span
-              data-testid={`category-expand-${metadataField}`}
               style={{
                 cursor: "pointer",
                 display: "inline-block"
@@ -157,26 +152,18 @@ class Category extends React.Component {
             >
               {metadataField}
               {isExpanded ? (
-                <FaChevronDown
+                <FaChevronDown style={{ fontSize: 10, marginLeft: 5 }} />
-                  data-testclass="category-expand-is-expanded"
-                  style={{ fontSize: 10, marginLeft: 5 }}
-                />
               ) : (
-                <FaChevronRight
+                <FaChevronRight style={{ fontSize: 10, marginLeft: 5 }} />
-                  data-testclass="category-expand-is-not-expanded"
-                  style={{ fontSize: 10, marginLeft: 5 }}
-                />
               )}
             </span>
           </div>
           <Tooltip content="Use as color scale" position="bottom">
             <Button
-              data-testclass="colorby"
+              onClick={this.handleColorChange.bind(this)}
-              data-testid={`colorby-${metadataField}`}
-              onClick={this.handleColorChange}
               active={colorAccessor === metadataField}
               intent={colorAccessor === metadataField ? "primary" : "none"}
-              icon="tint"
+              icon={"tint"}
             />
           </Tooltip>
         </div>

client/src/components/categorical/occupancy.js

@@ -1,63 +0,0 @@
-// jshint esversion: 6
-import React from "react";
-import { connect } from "react-redux";
-import * as d3 from "d3";
-
-@connect()
-class Occupancy extends React.Component {
-  render() {
-    const { occupancy, colorScale, colorAccessor, schema, world } = this.props;
-    const width = 100;
-    const height = 11;
-
-    const categories = schema.annotations.obsByName[colorAccessor]?.categories;
-
-    const x = d3
-      .scaleLinear()
-      /* get all the keys d[1] as an array, then find the sum */
-      .domain([0, d3.sum(Array.from(occupancy, d => d[1]))])
-      .range([0, width]);
-
-    let currentOffset = 0;
-    const dfColumn = world.obsAnnotations.col(colorAccessor);
-    const categoryValues = dfColumn.summarize().categories;
-    const stacks = categoryValues.map(d => {
-      const o = occupancy.get(d);
-
-      const scaledValue = x(o);
-
-      const stackItem = {
-        key: d,
-        value: o || 0,
-        rectWidth: o ? scaledValue : 0,
-        offset: currentOffset,
-        fill: o ? colorScale(categories.indexOf(d)) : "rgb(255,255,255)"
-      };
-      currentOffset += o ? scaledValue : 0;
-      return stackItem;
-    });
-
-    return (
-      <svg
-        style={{
-          marginRight: 5,
-          width,
-          height
-        }}
-      >
-        {stacks.map(d => (
-          <rect
-            key={d.key}
-            width={d.rectWidth}
-            height={height}
-            x={d.offset}
-            title={d.metadataField}
-            fill={d.fill}
-          />
-        ))}
-      </svg>
-    );
-  }
-}
-
-export default Occupancy;

client/src/components/categorical/util.js

@@ -3,32 +3,9 @@
 // values is [ [optVal, optIdx], ...]
 // index is range array
 // return sorted index
-
+export default values =>
-import isNumber from "is-number";
+  values.sort((a, b) => {
-
-const sortedCategoryValues = values => {
-  /* this sort could be memoized for perf */
-
-  const strings = [];
-  const ints = [];
-
-  values.forEach(v => {
-    if (isNumber(v[0])) {
-      ints.push(v);
-    } else {
-      strings.push(v);
-    }
-  });
-
-  strings.sort((a, b) => {
     const textA = String(a[0]).toUpperCase();
     const textB = String(b[0]).toUpperCase();
     return textA < textB ? -1 : textA > textB ? 1 : 0;
   });
-
-  ints.sort((a, b) => +a[0] - +b[0]);
-
-  return ints.concat(strings);
-};
-
-export default sortedCategoryValues;

client/src/components/categorical/value.js

@@ -1,73 +1,60 @@
 // jshint esversion: 6
 import { connect } from "react-redux";
 import React from "react";
-import Occupancy from "./occupancy";
+import _ from "lodash";
-import { countCategoryValues2D } from "../../util/stateManager/worldUtil";
-import * as globals from "../../globals";
 
 @connect(state => ({
-  categoricalSelection: state.categoricalSelection,
+  categoricalSelectionState: state.controls.categoricalSelectionState,
-  colorScale: state.colors.scale,
+  colorScale: state.controls.colorScale,
-  colorAccessor: state.colors.colorAccessor,
+  colorAccessor: state.controls.colorAccessor,
-  schema: state.world?.schema,
+  schema: _.get(state.controls.world, "schema", null)
-  world: state.world
 }))
 class CategoryValue extends React.Component {
   toggleOff() {
-    const { dispatch, metadataField, categoryIndex } = this.props;
+    const { dispatch, metadataField, optionIndex } = this.props;
     dispatch({
       type: "categorical metadata filter deselect",
       metadataField,
-      categoryIndex
+      optionIndex
     });
   }
 
   toggleOn() {
-    const { dispatch, metadataField, categoryIndex } = this.props;
+    const { dispatch, metadataField, optionIndex } = this.props;
     dispatch({
       type: "categorical metadata filter select",
       metadataField,
-      categoryIndex
+      optionIndex
     });
   }
 
   render() {
     const {
-      categoricalSelection,
+      categoricalSelectionState,
       metadataField,
-      categoryIndex,
+      optionIndex,
       colorAccessor,
       colorScale,
       i,
-      schema,
+      schema
-      world
     } = this.props;
 
-    if (!categoricalSelection) return null;
+    if (!categoricalSelectionState) return null;
 
-    const category = categoricalSelection[metadataField];
+    const category = categoricalSelectionState[metadataField];
-    const selected = category.categorySelected[categoryIndex];
+    const selected = category.optionSelected[optionIndex];
-    const count = category.categoryCounts[categoryIndex];
+    const count = category.optionCount[optionIndex];
-    const value = category.categoryValues[categoryIndex];
+    const value = category.optionValue[optionIndex];
-    const displayString = String(
+    const displayString = String(category.optionValue[optionIndex]).valueOf();
-      category.categoryValues[categoryIndex]
-    ).valueOf();
 
     /* this is the color scale, so add swatches below */
-    const isColorBy = metadataField === colorAccessor;
+    const c = metadataField === colorAccessor;
     let categories = null;
-    let occupancy = null;
 
-    if (isColorBy && schema) {
+    if (c && schema) {
-      categories = schema.annotations.obsByName[colorAccessor]?.categories;
+      categories = _.filter(schema.annotations.obs, {
-    }
+        name: colorAccessor
-
+      })[0].categories;
-    if (colorAccessor && !isColorBy && categoricalSelection[colorAccessor]) {
-      occupancy = countCategoryValues2D(
-        metadataField,
-        colorAccessor,
-        world.obsAnnotations
-      );
     }
 
     return (
@@ -78,16 +65,12 @@ class CategoryValue extends React.Component {
           alignItems: "baseline",
           justifyContent: "space-between"
         }}
-        data-testclass="categorical-row"
       >
         <div
           style={{
             margin: 0,
             padding: 0,
-            userSelect: "none",
+            userSelect: "none"
-            width: globals.leftSidebarWidth - 130,
-            display: "flex",
-            justifyContent: "space-between"
           }}
         >
           <label className="bp3-control bp3-checkbox">
@@ -95,46 +78,22 @@ class CategoryValue extends React.Component {
               onChange={
                 selected ? this.toggleOff.bind(this) : this.toggleOn.bind(this)
               }
-              data-testclass="categorical-value-select"
-              data-testid={`categorical-value-select-${metadataField}-${displayString}`}
               checked={selected}
               type="checkbox"
             />
             <span className="bp3-control-indicator" />
-            <span
+            {displayString}
-              data-testid={`categorical-value-${metadataField}-${displayString}`}
-              data-testclass="categorical-value"
-            >
-              {displayString}
-            </span>
           </label>
-          <span style={{ flexShrink: 0 }}>
-            {colorAccessor &&
-            !isColorBy &&
-            categoricalSelection[colorAccessor] ? (
-              <Occupancy
-                occupancy={occupancy.get(
-                  category.categoryValues[categoryIndex]
-                )}
-                {...this.props}
-              />
-            ) : null}
-          </span>
         </div>
         <span>
-          <span
+          <span>{count}</span>
-            data-testclass="categorical-value-count"
-            data-testid={`categorical-value-count-${metadataField}-${displayString}`}
-          >
-            {count}
-          </span>
           <svg
             style={{
               marginLeft: 5,
               width: 11,
               height: 11,
               backgroundColor:
-                isColorBy && categories
+                c && categories
                   ? colorScale(categories.indexOf(value))
                   : "inherit"
             }}

client/src/components/continuous/continuous.js

@@ -9,10 +9,12 @@ import * as globals from "../../globals";
 import HistogramBrush from "../brushableHistogram";
 
 @connect(state => ({
-  obsAnnotations: state.world?.obsAnnotations,
+  ranges: _.get(state.controls.world, "summary.obs", null),
-  colorAccessor: state.colors.colorAccessor,
+  metadata: _.get(state.controls.world, "obsAnnotations", null),
-  colorScale: state.colors.scale,
+  colorAccessor: state.controls.colorAccessor,
-  schema: state.world?.schema
+  colorScale: state.controls.colorScale,
+  selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null),
+  schema: _.get(state.controls.world, "schema", null)
 }))
 class Continuous extends React.Component {
   constructor(props) {
@@ -27,18 +29,17 @@ class Continuous extends React.Component {
 
   handleColorAction(key) {
     return () => {
-      const { dispatch, obsAnnotations } = this.props;
+      const { dispatch, ranges } = this.props;
-      const summary = obsAnnotations.col(key).summarize();
       dispatch({
         type: "color by continuous metadata",
         colorAccessor: key,
-        rangeForColorAccessor: summary
+        rangeMaxForColorAccessor: ranges[key].range.max
       });
     };
   }
 
   render() {
-    const { obsAnnotations, schema } = this.props;
+    const { ranges, obsAnnotations, schema } = this.props;
     if (schema && !this.continuousChecked) {
       this.hasContinuous = _.some(
         schema.annotations.obs,
@@ -62,31 +63,24 @@ class Continuous extends React.Component {
             Continuous metadata
           </p>
         ) : null}
-        {obsAnnotations
+        {_.map(ranges, (value, key) => {
-          ? _.map(obsAnnotations.colIndex.keys(), key => {
+          const isColorField = key.includes("color") || key.includes("Color");
-              const isColorField =
+          zebra += 1;
-                key.includes("color") || key.includes("Color");
+          if (value.range && key !== "name" && !isColorField) {
-              if (key === "name" || isColorField) return null;
+            return (
-
+              <HistogramBrush
-              const summary = obsAnnotations.col(key).summarize();
+                key={key}
-              const nonFiniteExtent =
+                field={key}
-                summary.min === undefined || summary.max === undefined;
+                isObs
-              if (!summary.categorical && !nonFiniteExtent) {
+                zebra={zebra % 2 === 0}
-                zebra += 1;
+                fieldValues={obsAnnotations}
-                return (
+                ranges={value.range}
-                  <HistogramBrush
+                handleColorAction={this.handleColorAction(key).bind(this)}
-                    key={key}
+              />
-                    field={key}
+            );
-                    isObs
+          }
-                    zebra={zebra % 2 === 0}
+          return null;
-                    ranges={summary}
+        })}
-                    handleColorAction={this.handleColorAction(key).bind(this)}
-                  />
-                );
-              }
-              return null;
-            })
-          : null}
       </div>
     );
   }

client/src/components/continuousLegend/index.js

@@ -98,8 +98,8 @@ const continuous = (selectorId, colorscale, colorAccessor) => {
 };
 
 @connect(state => ({
-  colorAccessor: state.colors.colorAccessor,
+  colorAccessor: state.controls.colorAccessor,
-  colorScale: state.colors.scale,
+  colorScale: state.controls.colorScale,
   responsive: state.responsive
 }))
 class ContinuousLegend extends React.Component {
@@ -112,7 +112,6 @@ class ContinuousLegend extends React.Component {
     const { colorAccessor, responsive, colorScale } = this.props;
     if (
       prevProps.colorAccessor !== colorAccessor ||
-      prevProps.colorScale !== colorScale ||
       prevProps.responsive.height !== responsive.height ||
       prevProps.responsive.width !== responsive.width
     ) {

client/src/components/framework/toasters.js

@@ -13,13 +13,6 @@ A "user" error - eg, bad input
 export const postUserErrorToast = message =>
   ErrorToastTopCenter.show({ message, intent: Intent.WARNING });
 
-/*
-A toast the user must dismiss manually, because they need to act on its information,
-ie., 8 bulk add genes out of 40 were bad. Manually see which ones and fix.
-*/
-export const keepAroundErrorToast = message =>
-  ErrorToastTopCenter.show({ message, timeout: 0, intent: Intent.WARNING });
-
 /*
 a hard network error
 */

client/src/components/geneExpression/cellSetButtons.js

@@ -1,8 +1,8 @@
 // jshint esversion: 6
 import React from "react";
-import { AnchorButton, Tooltip } from "@blueprintjs/core";
+import _ from "lodash";
+import { Button, Tooltip } from "@blueprintjs/core";
 import { connect } from "react-redux";
-import { World } from "../../util/stateManager";
 
 @connect()
 class CellSetButton extends React.Component {
@@ -14,11 +14,7 @@ class CellSetButton extends React.Component {
       eitherCellSetOneOrTwo
     } = this.props;
 
-    // Reducer and components assume that value will be null if
+    const set = _.map(crossfilter.allFiltered(), "name");
-    // no selection made.  World..getSelectedByIndex() returns a
-    // zero length TypedArray when nothing is selected.
-    let set = World.getSelectedByIndex(crossfilter);
-    if (set.length === 0) set = null;
 
     if (!differential.diffExp) {
       /* diffexp needs to be cleared before we store a new set */
@@ -32,28 +28,22 @@ class CellSetButton extends React.Component {
   render() {
     const { differential, eitherCellSetOneOrTwo } = this.props;
     const cellListName = `celllist${eitherCellSetOneOrTwo}`;
-    let cells_selected = differential[cellListName]
-      ? differential[cellListName].length
-      : 0;
     return (
       <Tooltip
         content="Save current selection for differential expression computation"
         position="top"
       >
-        <AnchorButton
+        <Button
           style={{ marginRight: 10 }}
           type="button"
-          disabled={differential.diffExp}
           onClick={this.set.bind(this)}
-          data-testid={`cellset-button-${eitherCellSetOneOrTwo}`}
         >
           {eitherCellSetOneOrTwo}
           {": "}
-          <span data-testid={`cellset-count-${eitherCellSetOneOrTwo}`}>
+          {differential[cellListName]
-            {cells_selected}
+            ? `${differential[cellListName].length} cells`
-          </span>
+            : "0 cells"}
-          {" cells"}
+        </Button>
-        </AnchorButton>
       </Tooltip>
     );
   }

client/src/components/geneExpression/expressionButtons.js

@@ -9,8 +9,9 @@ import CellSetButton from "./cellSetButtons";
 
 @connect(state => ({
   differential: state.differential,
-  world: state.world,
+  world: state.controls.world,
-  crossfilter: state.crossfilter
+  crossfilter: state.controls.crossfilter,
+  selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null)
 }))
 class Expression extends React.Component {
   constructor(props) {
@@ -67,8 +68,6 @@ class Expression extends React.Component {
               style={{ marginTop: 10 }}
               disabled={!haveBothCellSets}
               intent="primary"
-              data-testid="diffexp-button"
-              loading={differential.loading}
               fill
               type="button"
               onClick={this.computeDiffExp.bind(this)}

client/src/components/geneExpression/index.js

@@ -3,65 +3,28 @@
 
 import React from "react";
 import _ from "lodash";
-import fuzzysort from "fuzzysort";
+import * as d3 from "d3";
-
 import { connect } from "react-redux";
-import {
+import { Button, Tooltip } from "@blueprintjs/core";
-  MenuItem,
-  Button,
-  FormGroup,
-  InputGroup,
-  ControlGroup
-} from "@blueprintjs/core";
-import { Suggest } from "@blueprintjs/select";
 import HistogramBrush from "../brushableHistogram";
 import * as globals from "../../globals";
 import actions from "../../actions";
-import {
+import { postUserErrorToast } from "../framework/toasters";
-  postUserErrorToast,
-  keepAroundErrorToast
-} from "../framework/toasters";
 import ExpressionButtons from "./expressionButtons";
 
-const renderGene = (fuzzySortResult, { handleClick, modifiers, query }) => {
-  if (!modifiers.matchesPredicate) {
-    return null;
-  }
-  /* the fuzzysort wraps the object with other properties, like a score */
-  const geneName = fuzzySortResult.target;
-
-  return (
-    <MenuItem
-      active={modifiers.active}
-      disabled={modifiers.disabled}
-      data-testid={`suggest-menu-item-${geneName}`}
-      // Use of annotations in this way is incorrect and dataset specific.
-      // See https://github.com/chanzuckerberg/cellxgene/issues/483
-      // label={gene.n_counts}
-      key={geneName}
-      onClick={g =>
-        /* this fires when user clicks a menu item */
-        handleClick(g)
-      }
-      text={geneName}
-    />
-  );
-};
-
-const filterGenes = (query, genes) =>
-  /* fires on load, once, and then for each character typed into the input */
-  fuzzysort.go(query, genes, {
-    limit: 5,
-    threshold: -10000 // don't return bad results
-  });
-
 @connect(state => {
+  const metadata = _.get(state.controls.world, "obsAnnotations", null);
+  const ranges = _.get(state.controls.world, "summary.obs", null);
+  const initializeRanges = _.get(state.controls.world, "summary.obs");
+
   return {
-    obsAnnotations: state.world?.obsAnnotations,
+    ranges,
+    metadata,
+    initializeRanges,
     userDefinedGenes: state.controls.userDefinedGenes,
-    userDefinedGenesLoading: state.controls.userDefinedGenesLoading,
+    world: state.controls.world,
-    world: state.world,
+    colorAccessor: state.controls.colorAccessor,
-    colorAccessor: state.colors.colorAccessor,
+    allGeneNames: state.controls.allGeneNames,
     differential: state.differential
   };
 })
@@ -69,107 +32,41 @@ class GeneExpression extends React.Component {
   constructor(props) {
     super(props);
     this.state = {
-      bulkAdd: "",
+      gene: ""
-      tab: "autosuggest"
     };
   }
 
-  placeholderGeneNames() {
+  keyPress(e) {
-    /*
+    if (e.keyCode === 13) {
-    return a string containing gene name suggestions for use as a user hint.
+      this.handleClick();
-    Eg.,    Apod, Cd74, ...
-    Will return a max of 3 genes, totalling 15 characters in length.
-    Randomly selects gene names.
-
-    NOTE: the random selection means it will re-render constantly.
-    */
-    const { world } = this.props;
-    const { varAnnotations } = world;
-    const geneNames = varAnnotations.col("name").asArray();
-    if (geneNames.length > 0) {
-      const placeholder = [];
-      let len = geneNames.length;
-      const maxGeneNameCount = 3;
-      const maxStrLength = 15;
-      len = len < maxGeneNameCount ? len : maxGeneNameCount;
-      for (let i = 0, strLen = 0; i < len && strLen < maxStrLength; i += 1) {
-        const deal = Math.floor(Math.random() * geneNames.length);
-        const geneName = geneNames[deal];
-        placeholder.push(geneName);
-        strLen += geneName.length + 2; // '2' is the length of a comma and space
-      }
-      placeholder.push("...");
-      return placeholder.join(", ");
     }
-    // default - should never happen.
-    return "Apod, Cd74, ...";
   }
 
-  handleClick(g) {
+  handleClick() {
     const { world, dispatch, userDefinedGenes } = this.props;
-    const gene = g.target;
+    const { gene } = this.state;
+
     if (userDefinedGenes.indexOf(gene) !== -1) {
       postUserErrorToast("That gene already exists");
     } else if (userDefinedGenes.length > 15) {
       postUserErrorToast(
         "That's too many genes, you can have at most 15 user defined genes"
       );
-    } else if (world.varAnnotations.col("name").indexOf(gene) === undefined) {
+    } else if (!_.find(world.varAnnotations, { name: gene })) {
       postUserErrorToast("That doesn't appear to be a valid gene name.");
     } else {
-      dispatch({ type: "single user defined gene start" });
+      dispatch(actions.requestUserDefinedGene(gene));
-      dispatch(actions.requestUserDefinedGene(gene)).then(
+      dispatch({
-        () => dispatch({ type: "single user defined gene complete" }),
+        type: "user defined gene",
-        () => dispatch({ type: "single user defined gene error" })
+        data: gene
-      );
+      });
+      this.setState({ gene: "" });
     }
   }
 
-  handleBulkAddClick() {
-    const { world, dispatch, userDefinedGenes } = this.props;
-    const { bulkAdd } = this.state;
-
-    /*
-      test:
-      Apod,,, Cd74,,    ,,,    Foo,    Bar-2,,
-    */
-    if (bulkAdd !== "") {
-      const genes = _.pull(_.uniq(bulkAdd.split(/[ ,]+/)), "");
-
-      dispatch({ type: "bulk user defined gene start" });
-      Promise.all(
-        genes.map(gene => {
-          if (gene.length === 0) {
-            return keepAroundErrorToast("Must enter a gene name.");
-          }
-          if (userDefinedGenes.indexOf(gene) !== -1) {
-            return keepAroundErrorToast("That gene already exists");
-          }
-          if (world.varAnnotations.col("name").indexOf(gene) === undefined) {
-            return keepAroundErrorToast(
-              `${gene} doesn't appear to be a valid gene name.`
-            );
-          }
-          return dispatch(actions.requestUserDefinedGene(gene));
-        })
-      ).then(
-        () => dispatch({ type: "bulk user defined gene complete" }),
-        () => dispatch({ type: "bulk user defined gene error" })
-      );
-    }
-
-    this.setState({ bulkAdd: "" });
-  }
-
   render() {
-    const {
+    const { world, userDefinedGenes, differential } = this.props;
-      world,
+    const { gene } = this.state;
-      userDefinedGenes,
-      userDefinedGenesLoading,
-      differential
-    } = this.props;
-
-    const { tab, bulkAdd } = this.state;
 
     return (
       <div>
@@ -187,123 +84,43 @@ class GeneExpression extends React.Component {
             Selected Genes
           </p>
           <div
-            style={{
+            style={{ padding: globals.leftSidebarSectionPadding }}
-              padding: globals.leftSidebarSectionPadding
+            className="bp3-control-group"
-            }}
           >
-            <Button
+            <div className="bp3-input-group bp3-fill">
-              active={tab === "autosuggest"}
+              <input
-              style={{ marginRight: 5 }}
+                onKeyDown={this.keyPress.bind(this)}
-              minimal
+                onChange={e => {
-              small
+                  this.setState({ gene: e.target.value });
-              data-testid="tab-autosuggest"
-              onClick={() => {
-                this.setState({ tab: "autosuggest" });
-              }}
-            >
-              Autosuggest
-            </Button>
-            <Button
-              active={tab === "bulkadd"}
-              minimal
-              small
-              data-testid="section-bulk-add"
-              onClick={() => {
-                this.setState({ tab: "bulkadd" });
-              }}
-            >
-              Bulk add genes
-            </Button>
-          </div>
-
-          {tab === "autosuggest" ? (
-            <ControlGroup
-              style={{
-                paddingLeft: globals.leftSidebarSectionPadding,
-                paddingBottom: globals.leftSidebarSectionPadding
-              }}
-            >
-              <Suggest
-                resetOnSelect
-                closeOnSelect
-                resetOnClose
-                itemDisabled={
-                  userDefinedGenesLoading ? () => true : () => false
-                }
-                noResults={<MenuItem disabled text="No matching genes." />}
-                onItemSelect={g => {
-                  /* this happens on 'enter' */
-                  this.handleClick(g);
                 }}
-                inputProps={{ "data-testid": "gene-search" }}
+                value={gene}
-                inputValueRenderer={g => {
+                type="text"
-                  return "";
+                className="bp3-input"
-                }}
+                placeholder="Enter a gene name"
-                itemListPredicate={filterGenes}
+                style={{ paddingRight: 94 }}
-                itemRenderer={renderGene.bind(this)}
-                items={
-                  world && world.varAnnotations
-                    ? world.varAnnotations.col("name").asArray()
-                    : ["No genes"]
-                }
-                popoverProps={{ minimal: true }}
               />
-              <Button
+            </div>
-                className="bp3-button bp3-intent-primary"
+            <Tooltip
-                data-testid={"add-gene"}
+              content="Add a gene to see its expression levels"
-                loading={userDefinedGenesLoading}
+              position="bottom"
-              >
+            >
+              <Button intent="primary" onClick={this.handleClick.bind(this)}>
                 Add
               </Button>
-            </ControlGroup>
+            </Tooltip>
-          ) : null}
+          </div>
-          {tab === "bulkadd" ? (
-            <div style={{ paddingLeft: globals.leftSidebarSectionPadding }}>
-              <form
-                onSubmit={e => {
-                  e.preventDefault();
-                  this.handleBulkAddClick();
-                }}
-              >
-                <FormGroup
-                  helperText="Add a list of genes (comma delimited)"
-                  labelFor="text-input-bulk-add"
-                >
-                  <ControlGroup>
-                    <InputGroup
-                      onChange={e => {
-                        this.setState({ bulkAdd: e.target.value });
-                      }}
-                      id="text-input-bulk-add"
-                      data-testid="input-bulk-add"
-                      placeholder={this.placeholderGeneNames()}
-                      value={bulkAdd}
-                    />
-                    <Button
-                      intent="primary"
-                      onClick={this.handleBulkAddClick.bind(this)}
-                      loading={userDefinedGenesLoading}
-                    >
-                      Add
-                    </Button>
-                  </ControlGroup>
-                </FormGroup>
-              </form>
-            </div>
-          ) : null}
           {world && userDefinedGenes.length > 0
             ? _.map(userDefinedGenes, (geneName, index) => {
-                const values = world.varData.col(geneName);
+                const values = world.varDataCache[geneName];
                 if (!values) {
                   return null;
                 }
-                const summary = values.summarize();
                 return (
                   <HistogramBrush
                     key={geneName}
                     field={geneName}
                     zebra={index % 2 === 0}
-                    ranges={summary}
+                    ranges={d3.extent(values)}
                     isUserDefined
                   />
                 );
@@ -322,18 +139,18 @@ class GeneExpression extends React.Component {
           <ExpressionButtons />
           {differential.diffExp
             ? _.map(differential.diffExp, (value, index) => {
-                const name = world.varAnnotations.at(value[0], "name");
+                const annotations = world.varAnnotations[value[0]];
-                const values = world.varData.col(name);
+                const { name } = annotations;
+                const values = world.varDataCache[name];
                 if (!values) {
                   return null;
                 }
-                const summary = values.summarize();
                 return (
                   <HistogramBrush
                     key={name}
                     field={name}
                     zebra={index % 2 === 0}
-                    ranges={summary}
+                    ranges={d3.extent(values)}
                     isDiffExp
                     logFoldChange={value[1]}
                     pval={value[2]}

client/src/components/graph/setupLasso.js

@@ -1,146 +0,0 @@
-// https://bl.ocks.org/pbeshai/8008075f9ce771ee8be39e8c38907570
-
-import * as d3 from "d3";
-
-const Lasso = () => {
-  const dispatch = d3.dispatch("start", "end", "cancel");
-
-  const polygonToPath = polygon =>
-    `M${polygon.map(d => d.join(",")).join("L")}`;
-
-  const distance = (pt1, pt2) =>
-    Math.sqrt((pt2[0] - pt1[0]) ** 2 + (pt2[1] - pt1[1]) ** 2);
-
-  // distance last point has to be to first point before it auto closes when mouse is released
-  const closeDistance = 75;
-
-  const lasso = svg => {
-    let lassoPolygon;
-    let lassoPath;
-    let closePath;
-
-    const handleDragStart = () => {
-      lassoPolygon = [d3.mouse(svg.node())]; // current x y of mouse within element
-
-      if (lassoPath) {
-        lassoPath.remove();
-      }
-
-      lassoPath = g
-        .append("path")
-        .attr("data-testid", "lasso-element")
-        .attr("fill", "#0bb")
-        .attr("fill-opacity", 0.1)
-        .attr("stroke", "#0bb")
-        .attr("stroke-dasharray", "3, 3");
-
-      closePath = g
-        .append("line")
-        .attr("x2", lassoPolygon[0][0])
-        .attr("y2", lassoPolygon[0][1])
-        .attr("stroke", "#0bb")
-        .attr("stroke-dasharray", "3, 3")
-        .attr("opacity", 0);
-
-      dispatch.call("start", lasso, lassoPolygon);
-    };
-
-    const handleDrag = () => {
-      const point = d3.mouse(svg.node());
-      lassoPolygon.push(point);
-      lassoPath.attr("d", polygonToPath(lassoPolygon));
-
-      // indicate if we are within closing distance
-      if (
-        distance(lassoPolygon[0], lassoPolygon[lassoPolygon.length - 1]) <
-        closeDistance
-      ) {
-        closePath
-          .attr("x1", point[0])
-          .attr("y1", point[1])
-          .attr("opacity", 1);
-      } else {
-        closePath.attr("opacity", 0);
-      }
-    };
-
-    const handleDragEnd = () => {
-      // remove the close path
-      closePath.remove();
-      closePath = null;
-
-      // succesfully closed
-      if (
-        distance(lassoPolygon[0], lassoPolygon[lassoPolygon.length - 1]) <
-        closeDistance
-      ) {
-        lassoPath.attr("d", `${polygonToPath(lassoPolygon)}Z`);
-        dispatch.call("end", lasso, lassoPolygon);
-
-        // otherwise cancel
-      } else {
-        lassoPath.remove();
-        lassoPath = null;
-        lassoPolygon = null;
-        dispatch.call("cancel");
-      }
-    };
-
-    // append a <g> with a rect
-    const g = svg.append("g").attr("class", "lasso-group");
-    const bbox = svg.node().getBoundingClientRect();
-    const area = g
-      .append("rect")
-      .attr("width", bbox.width)
-      .attr("height", bbox.height)
-      .attr("fill", "tomato")
-      .attr("opacity", 0);
-
-    const drag = d3
-      .drag()
-      .on("start", handleDragStart)
-      .on("drag", handleDrag)
-      .on("end", handleDragEnd);
-
-    area.call(drag);
-
-    lasso.reset = () => {
-      if (lassoPath) {
-        lassoPath.remove();
-        lassoPath = null;
-      }
-
-      lassoPolygon = null;
-      if (closePath) {
-        closePath.remove();
-        closePath = null;
-      }
-    };
-
-    lasso.move = polygon => {
-      if (polygon !== lassoPolygon || polygon.length !== lassoPolygon.length) {
-        lasso.reset();
-
-        lassoPolygon = polygon;
-        lassoPath = g
-          .append("path")
-          .attr("data-testid", "lasso-element")
-          .attr("fill", "#0bb")
-          .attr("fill-opacity", 0.1)
-          .attr("stroke", "#0bb")
-          .attr("stroke-dasharray", "3, 3");
-
-        lassoPath.attr("d", `${polygonToPath(lassoPolygon)}Z`);
-      }
-    };
-  };
-
-  lasso.on = (type, callback) => {
-    dispatch.on(type, callback);
-    return lasso;
-  };
-
-  return lasso;
-};
-
-export default Lasso;

client/src/components/graph/setupSVGandBrush.js

@@ -1,7 +1,6 @@
 // jshint esversion: 6
 import * as d3 from "d3";
 import styles from "./graph.css";
-import Lasso from "./setupLasso";
 
 /******************************************
 *******************************************
@@ -10,53 +9,32 @@ import Lasso from "./setupLasso";
 ******************************************/
 
 export default (
-  selectionToolType,
+  handleBrushSelectAction,
-  handleStartAction,
+  handleBrushDeselectAction,
-  handleDragAction,
-  handleEndAction,
-  handleCancelAction,
   responsive,
   graphPaddingRight
 ) => {
   const svg = d3
     .select("#graphAttachPoint")
     .append("svg")
-    .attr("data-testid", "layout-overlay")
     .attr("width", responsive.width - graphPaddingRight)
     .attr("height", responsive.height)
     .attr("class", `${styles.graphSVG}`);
 
-  if (selectionToolType === "brush") {
+  const brush = d3
-    const brush = d3
+    .brush()
-      .brush()
+    .extent([[0, 0], [responsive.width - graphPaddingRight, responsive.height]])
-      .extent([
+    .on("brush", handleBrushSelectAction)
-        [0, 0],
+    .on("end", handleBrushDeselectAction);
-        [responsive.width - graphPaddingRight, responsive.height]
-      ])
-      .on("start", handleStartAction)
-      .on("brush", handleDragAction)
-      // FYI, brush doesn't generate cancel
-      .on("end", handleEndAction);
 
-    const brushContainer = svg
+  const brushContainer = svg
-      .append("g")
+    .append("g")
-      .attr("class", "graph_brush")
+    .attr("class", "graph_brush")
-      .call(brush);
+    .call(brush);
 
-    return { svg, container: brushContainer, tool: brush };
+  return {
-  }
+    svg,
-
+    brushContainer,
-  if (selectionToolType === "lasso") {
+    brush
-    const lasso = Lasso()
+  };
-      .on("end", handleEndAction)
-      // FYI, Lasso doesn't generate drag
-      .on("start", handleStartAction)
-      .on("cancel", handleCancelAction);
-
-    const lassoContainer = svg.call(lasso);
-
-    return { svg, container: lassoContainer, tool: lasso };
-  }
-
-  throw new Error("unknown graph selection tool");
 };

client/src/components/graph/util.js

@@ -0,0 +1,26 @@
+// jshint esversion: 6
+// createExpressionsCountsMap () {
+//
+//   const CHANGE_ME_MAGIC_GENE_INDEX = 5;
+//
+//   const expressionsCountsMap = {};
+//
+//   /* currently selected gene */
+//   expressionsCountsMap.geneName = this.state.expressions.data.genes[3];
+//
+//   let maxExpressionValue = 0;
+//
+//   /* create map of expressions for every cell */
+//   this.state.expressions.data.cells.map((c) => {
+//     /* cellname = 234 */
+//     expressionsCountsMap[c.cellname] = c["e"][CHANGE_ME_MAGIC_GENE_INDEX];
+//     /* collect the maximum value as we iterate */
+//     if (c["e"][CHANGE_ME_MAGIC_GENE_INDEX] > maxExpressionValue) {
+//       maxExpressionValue = c["e"][CHANGE_ME_MAGIC_GENE_INDEX]
+//     }
+//   })
+//
+//   expressionsCountsMap.maxValue = maxExpressionValue;
+//
+//   return expressionsCountsMap;
+// }

client/src/components/leftsidebar.js

@@ -1,4 +1,5 @@
 // jshint esversion: 6
+import _ from "lodash";
 import React from "react";
 import { connect } from "react-redux";
 import Categorical from "./categorical/categorical";
@@ -9,7 +10,7 @@ import DynamicScatterplot from "./scatterplot/scatterplot";
 
 @connect(state => ({
   responsive: state.responsive,
-  datasetTitle: state.config?.displayNames?.dataset,
+  datasetTitle: _.get(state.config, "displayNames.dataset"),
   scatterplotXXaccessor: state.controls.scatterplotXXaccessor,
   scatterplotYYaccessor: state.controls.scatterplotYYaccessor
 }))
@@ -39,7 +40,6 @@ class LeftSideBar extends React.Component {
         }}
       >
         <p
-          data-testid="header"
           style={{
             position: "fixed",
             top: globals.cellxgeneTitleTopPadding,

client/src/components/scatterplot/scatterplot.js

@@ -19,30 +19,30 @@ import _drawPoints from "./drawPointsRegl";
 import scaleLinear from "../../util/scaleLinear";
 
 import { margin, width, height } from "./util";
-import finiteExtent from "../../util/finiteExtent";
+import { kvCache } from "../../util/stateManager";
 
 @connect(state => {
-  const { world, crossfilter } = state;
+  const {
-  const { scatterplotXXaccessor, scatterplotYYaccessor } = state.controls;
+    world,
+    crossfilter,
+    scatterplotXXaccessor,
+    scatterplotYYaccessor
+  } = state.controls;
   const expressionX =
-    world &&
+    world && scatterplotXXaccessor
-    scatterplotXXaccessor &&
+      ? kvCache.get(world.varDataCache, scatterplotXXaccessor)
-    world.varData.hasCol(scatterplotXXaccessor)
-      ? world.varData.col(scatterplotXXaccessor).asArray()
       : null;
   const expressionY =
-    world &&
+    world && scatterplotYYaccessor
-    scatterplotYYaccessor &&
+      ? kvCache.get(world.varDataCache, scatterplotYYaccessor)
-    world.varData.hasCol(scatterplotYYaccessor)
-      ? world.varData.col(scatterplotYYaccessor).asArray()
       : null;
 
   return {
     world,
 
-    colorRGB: state.colors.rgb,
+    colorRGB: state.controls.colorRGB,
-    colorScale: state.colors.scale,
+    colorAccessor: state.controls.colorAccessor,
-    colorAccessor: state.colors.colorAccessor,
+    colorScale: state.controls.colorScale,
 
     // Accessors are var/gene names (strings)
     scatterplotXXaccessor,
@@ -54,7 +54,9 @@ import finiteExtent from "../../util/finiteExtent";
     expressionX,
     expressionY,
 
-    crossfilter
+    crossfilter,
+    // updated whenever the crossfilter selection is updated
+    selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null)
   };
 })
 class Scatterplot extends React.Component {
@@ -62,16 +64,11 @@ class Scatterplot extends React.Component {
     super(props);
     this.count = 0;
     this.axes = false;
-    this.renderCache = {
-      positions: null,
-      colors: null,
-      sizes: null,
-      xScale: null,
-      yScale: null
-    };
     this.state = {
       svg: null,
-      minimized: null
+      minimized: null,
+      xScale: null,
+      yScale: null
     };
   }
 
@@ -83,7 +80,6 @@ class Scatterplot extends React.Component {
     if (svg && expressionX && expressionY) {
       scales = Scatterplot.setupScales(expressionX, expressionY);
       this.drawAxesSVG(scales.xScale, scales.yScale, svg);
-      this.renderCache = { ...this.renderCache, ...scales };
     }
 
     const camera = _camera(this.reglCanvas, { scale: true, rotate: false });
@@ -116,6 +112,8 @@ class Scatterplot extends React.Component {
       pointBuffer,
       colorBuffer,
       svg,
+      xScale: scales ? scales.xScale : null,
+      yScale: scales ? scales.yScale : null,
       reglRender,
       camera,
       drawPoints
@@ -134,6 +132,8 @@ class Scatterplot extends React.Component {
     } = this.props;
     const {
       reglRender,
+      xScale,
+      yScale,
       regl,
       pointBuffer,
       colorBuffer,
@@ -144,13 +144,17 @@ class Scatterplot extends React.Component {
     } = this.state;
 
     if (
-      scatterplotXXaccessor !== prevProps.scatterplotXXaccessor || // was CLU now FTH1 etc
+      world &&
+      svg &&
+      xScale &&
+      yScale &&
+      scatterplotXXaccessor &&
+      scatterplotYYaccessor &&
+      (scatterplotXXaccessor !== prevProps.scatterplotXXaccessor || // was CLU now FTH1 etc
       scatterplotYYaccessor !== prevProps.scatterplotYYaccessor || // was CLU now FTH1 etc
-      world !== prevProps.world // shape or clip of world changed
+        !this.axes) // clicked off the tab and back again, rerender
     ) {
-      const scales = Scatterplot.setupScales(expressionX, expressionY);
+      this.drawAxesSVG(xScale, yScale, svg);
-      this.drawAxesSVG(scales.xScale, scales.yScale, svg);
-      this.renderCache = { ...this.renderCache, ...scales };
     }
 
     if (reglRender && this.reglRenderState === "rendering") {
@@ -167,51 +171,35 @@ class Scatterplot extends React.Component {
       expressionX &&
       expressionY &&
       scatterplotXXaccessor &&
-      scatterplotYYaccessor
+      scatterplotYYaccessor &&
+      xScale &&
+      yScale
     ) {
-      const { renderCache } = this;
-      const { xScale, yScale } = this.renderCache;
       const cellCount = expressionX.length;
+      const positionsBuf = new Float32Array(2 * cellCount);
+      const colorsBuf = new Float32Array(3 * cellCount);
+      const sizesBuf = new Float32Array(cellCount);
 
-      // Points change when expressionX or expressionY change.
+      const glScaleX = scaleLinear([0, width], [-0.95, 0.95]);
-      if (
+      const glScaleY = scaleLinear([0, height], [-1, 1]);
-        !renderCache.positions ||
+
-        expressionX !== prevProps.expressionX ||
+      /*
-        expressionY !== prevProps.expressionY
+        Construct Vectors
-      ) {
+      */
-        if (!renderCache.positions) {
+      for (let i = 0; i < cellCount; i += 1) {
-          renderCache.positions = new Float32Array(2 * cellCount);
+        positionsBuf[2 * i] = glScaleX(xScale(expressionX[i]));
-        }
+        positionsBuf[2 * i + 1] = glScaleY(yScale(expressionY[i]));
-        const glScaleX = scaleLinear([0, width], [-0.95, 0.95]);
-        const glScaleY = scaleLinear([0, height], [-1, 1]);
-        for (let i = 0, { positions } = renderCache; i < cellCount; i += 1) {
-          positions[2 * i] = glScaleX(xScale(expressionX[i]));
-          positions[2 * i + 1] = glScaleY(yScale(expressionY[i]));
-        }
-        pointBuffer({ data: renderCache.positions, dimension: 2 });
       }
 
-      // Colors for each point - change only when props.colorsRGB change.
+      for (let i = 0; i < cellCount; i += 1) {
-      if (!renderCache.colors || colorRGB !== prevProps.colorRGB) {
+        colorsBuf.set(colorRGB[i], 3 * i);
-        if (!renderCache.colors) {
-          renderCache.colors = new Float32Array(3 * cellCount);
-        }
-        for (let i = 0, { colors } = renderCache; i < cellCount; i += 1) {
-          colors.set(colorRGB[i], 3 * i);
-        }
-        colorBuffer({ data: renderCache.colors, dimension: 3 });
       }
 
-      // Sizes for each point - updates are triggered only when selected
+      crossfilter.fillByIsFiltered(sizesBuf, 4, 0.2);
-      // obs change
-      if (!renderCache.sizes || crossfilter !== prevProps.crossfilter) {
-        if (!renderCache.sizes) {
-          renderCache.sizes = new Float32Array(cellCount);
-        }
-        crossfilter.fillByIsSelected(renderCache.sizes, 4, 0.2);
-        sizeBuffer({ data: renderCache.sizes, dimension: 1 });
-      }
 
+      pointBuffer({ data: positionsBuf, dimension: 2 });
+      colorBuffer({ data: colorsBuf, dimension: 3 });
+      sizeBuffer({ data: sizesBuf, dimension: 1 });
       this.count = cellCount;
 
       regl._refresh();
@@ -224,16 +212,26 @@ class Scatterplot extends React.Component {
         camera
       );
     }
+
+    if (
+      expressionX &&
+      expressionY &&
+      (scatterplotXXaccessor !== prevProps.scatterplotXXaccessor || // was CLU now FTH1 etc
+        scatterplotYYaccessor !== prevProps.scatterplotYYaccessor)
+    ) {
+      const scales = Scatterplot.setupScales(expressionX, expressionY);
+      this.setState(scales);
+    }
   }
 
   static setupScales(expressionX, expressionY) {
     const xScale = d3
       .scaleLinear()
-      .domain(finiteExtent(expressionX))
+      .domain(d3.extent(expressionX))
       .range([0, width]);
     const yScale = d3
       .scaleLinear()
-      .domain(finiteExtent(expressionY))
+      .domain(d3.extent(expressionY))
       .range([height, 0]);
 
     return {
@@ -264,15 +262,9 @@ class Scatterplot extends React.Component {
 
     // the axes are much cleaner and easier now. No need to rotate and orient
     // the axis, just call axisBottom, axisLeft etc.
-    const xAxis = d3
+    const xAxis = d3.axisBottom().scale(xScale);
-      .axisBottom()
-      .ticks(7)
-      .scale(xScale);
 
-    const yAxis = d3
+    const yAxis = d3.axisLeft().scale(yScale);
-      .axisLeft()
-      .ticks(7)
-      .scale(yScale);
 
     // adding axes is also simpler now, just translate x-axis to (0,height)
     // and it's alread defined to be a bottom axis.
@@ -345,12 +337,14 @@ class Scatterplot extends React.Component {
           <Button
             type="button"
             minimal
-            data-testid="clear-scatterplot"
+            onClick={() => {
-            onClick={() =>
               dispatch({
                 type: "clear scatterplot"
-              })
+              });
-            }
+              dispatch({
+                type: "reset colorscale"
+              });
+            }}
           >
             remove
           </Button>
@@ -366,7 +360,6 @@ class Scatterplot extends React.Component {
           <canvas
             width={width}
             height={height}
-            data-testid="scatterplot"
             style={{
               marginLeft: margin.left - 7,
               marginTop: margin.top

client/src/components/scatterplot/setupScatterplot.js

@@ -14,7 +14,6 @@ const setupScatterplot = (width, height, margin) => {
     .append("svg")
     .attr("width", width + margin.left + margin.right)
     .attr("height", height + margin.top + margin.bottom)
-    .attr("data-testid", "scatterplot-svg")
     .append("g")
     .attr("transform", `translate(${margin.left},${margin.top})`);
 

client/src/globals.js

@@ -1,4 +1,32 @@
-import { Colors } from "@blueprintjs/core";
+// jshint esversion: 6
+/* these will be either (preferably) specified or inferred */
+export const categories = [
+  "Sample.type",
+  "Selection",
+  "Location",
+  "Sample.name",
+  "Class",
+  "Neoplastic"
+];
+export const continuous = [
+  "Total_reads",
+  "Unique_reads",
+  "Unique_reads_percent",
+  "ERCC_reads",
+  "Non_ERCC_reads",
+  "ERCC_to_non_ERCC",
+  "Genes_detected",
+  "Multimapping_reads_percent",
+  "Splice_sites_AT.AC",
+  "Splice_sites_Annotated",
+  "Splice_sites_GC.AG",
+  "Splice_sites_GT.AG",
+  "Splice_sites_non_canonical",
+  "Splice_sites_total",
+  "Unmapped_mismatch",
+  "Unmapped_other",
+  "Unmapped_short"
+];
 
 /* if a categorical metadata field has more options than this, truncate */
 export const maxCategoricalOptionsToDisplay = 100;
@@ -16,8 +44,8 @@ export const configDefaults = {
 };
 
 /* colors */
-export const blue = Colors.BLUE3;
+export const blue = "#4a90e2";
-export const linkBlue = Colors.BLUE5;
+export const hcaBlue = "#1c7cc7";
 export const lightestGrey = "rgb(249,249,249)";
 export const lighterGrey = "rgb(245,245,245)";
 export const lightGrey = "rgb(211,211,211)";
@@ -29,7 +57,6 @@ export const brightBlue = "#4a90e2";
 export const brightGreen = "#A2D729";
 export const darkGreen = "#448C4D";
 
-export const nonFiniteCellColor = lightGrey;
 export const defaultCellColor = "rgb(0,0,0,1)";
 
 /* typography constants */

client/src/middleware/updateCellColors.js

@@ -0,0 +1,127 @@
+// jshint esversion: 6
+import _ from "lodash";
+import * as d3 from "d3";
+import {
+  interpolateViridis,
+  interpolateSpectral,
+  interpolateRainbow,
+  interpolateBlues,
+  interpolateCool
+} from "d3-scale-chromatic";
+import * as globals from "../globals";
+import parseRGB from "../util/parseRGB";
+
+/*
+  https://medium.com/@jacobp100/you-arent-using-redux-middleware-enough-94ffe991e6
+  storeInstance =>
+    functionToCallWithAnActionThatWillSendItToTheNextMiddleware =>
+    actionThatDispatchWasCalledWith =>
+    valueToUseAsTheReturnValueOfTheDispatchCall
+*/
+
+/*
+  What this file does:
+
+  1. fire a filter action anywhere in the app
+  2. ** this middleware checks to see the state of all the currently selected filters,
+     including the new one
+  3. ** create updated selection from a copy of all the cells presently on the client
+     (this may be a subset of 'all')
+  4. ** append that new selection to the action so that it magically appears in the reducer
+     just because the action was fired
+
+  This is nice because we keep a lot of filtering business logic centralized
+  (what it means in practice to be selected)
+*/
+
+const updateCellColorsMiddleware = store => next => action => {
+  const s = store.getState();
+
+  /*
+  this is a hardcoded map of the things we need to keep an eye on and update
+  global cell selection in response to
+  */
+  const filterJustChanged =
+    action.type === "color by expression" ||
+    action.type === "color by continuous metadata" ||
+    action.type === "color by categorical metadata";
+
+  if (!filterJustChanged || !s.controls.world.obsAnnotations) {
+    return next(
+      action
+    ); /* if the cells haven't loaded or the action wasn't a color change, bail */
+  }
+
+  const { obsAnnotations } = s.controls.world;
+  let colorScale;
+  const colorsByName = new Array(obsAnnotations.length);
+  const colorsByRGB = new Array(obsAnnotations.length);
+
+  /*
+  in plain language...
+   (a) once the cells have loaded.
+   (b) each time a user changes a color control we need to update cellsMetadata colors
+  This is available to all the draw functions as world.colorName[index] or world.colorRGB[index]
+  */
+
+  if (action.type === "color by categorical metadata") {
+    const categories = _.filter(s.controls.world.schema.annotations.obs, {
+      name: action.colorAccessor
+    })[0].categories;
+
+    colorScale = d3
+      .scaleSequential(interpolateRainbow)
+      .domain([0, categories.length]);
+
+    for (let i = 0; i < obsAnnotations.length; i += 1) {
+      const obs = obsAnnotations[i];
+      const c = colorScale(categories.indexOf(obs[action.colorAccessor]));
+      colorsByName[i] = c;
+      colorsByRGB[i] = parseRGB(c);
+    }
+  }
+
+  if (action.type === "color by continuous metadata") {
+    colorScale = d3
+      .scaleLinear()
+      .domain([0, action.rangeMaxForColorAccessor])
+      .range([1, 0]);
+
+    for (let i = 0; i < obsAnnotations.length; i += 1) {
+      const obs = obsAnnotations[i];
+      const c = interpolateCool(colorScale(obs[action.colorAccessor]));
+      colorsByName[i] = c;
+      colorsByRGB[i] = parseRGB(c);
+    }
+  }
+
+  if (action.type === "color by expression") {
+    const { gene, data } = action;
+    const expression = data[gene]; // Float32Array
+    colorScale = d3
+      .scaleLinear()
+      .domain([_.min(expression), _.max(expression)])
+      .range([
+        1,
+        0
+      ]); /* invert viridis... probably pass this scale through to others */
+
+    for (let i = 0, len = expression.length; i < len; i += 1) {
+      const c = interpolateCool(colorScale(expression[i]));
+      colorsByName[i] = c;
+      colorsByRGB[i] = parseRGB(c);
+    }
+  }
+
+  /*
+  append the result of all the filters to the action the user just triggered
+  */
+  const modifiedAction = Object.assign({}, action, {
+    colors: { name: colorsByName, rgb: colorsByRGB },
+    colorScale
+  });
+
+  return next(modifiedAction);
+};
+
+export default updateCellColorsMiddleware;

client/src/middleware/updateURLMiddleware.js

@@ -0,0 +1,87 @@
+// jshint esversion: 6
+// import uri from "urijs";
+
+/*
+NOTE: file currently not used, but retained as we expect to reinstate features in this
+area shortly.
+*/
+
+/*
+  https://medium.com/@jacobp100/you-arent-using-redux-middleware-enough-94ffe991e6
+  storeInstance
+    => functionToCallWithAnActionThatWillSendItToTheNextMiddleware
+    => actionThatDispatchWasCalledWith
+    => valueToUseAsTheReturnValueOfTheDispatchCall
+*/
+
+const updateURLMiddleware = (/* store */) => next => action => {
+  // const oldState = store.getState();
+  const nextAction = next(action);
+
+  if (action.type === "url changed") {
+    /* we don't handle pop state here - we handle it in the url reducer */
+    return nextAction;
+  }
+
+  // const state = store.getState();
+
+  /************************************************************************
+      *************************************************************************
+      1. Redux app state just changed. Clear URL, and then update it.
+      1a. We get the whole state tree to construct the url!
+      1b. But (see reducers/url.js) we try to centralize it because...
+      1c. ...the back button / initial load case ('url changed' return above)
+      means that we have to listen for 'url changed' and construct state
+      from the browser
+      *************************************************************************
+      ************************************************************************/
+
+  // const oldURI = URI(window.location.href)
+  // const newURI = URI(oldURI).setQuery({})
+
+  // if (window.location.search === "") {
+  //   newURL = uri.addQuery(category, value).toString(); /* #1 */
+  // } else if (uri.hasQuery(category, value) || uri.hasQuery(category, value, true)) { /* true param here means check arrays as well http://medialize.github.io/URI.js/docs.html#search-has */
+  //   newURL = uri.removeQuery(category, value).toString(); /* #4 */
+  // } else {
+  //   newURL = uri.addQuery(category, value).toString(); /* #2 & #3 are handled by URI */
+  // }
+  //
+  // window.history.pushState("", "", newURL)
+
+  //
+  // // Internal helper for working with URIs
+  // const oldURI = new URI(window.location.href);
+  // const newURI = new URI(oldURI).setQueryData({});
+  //
+  // newURI.setPath('/foo/bar');
+  //
+  // // Set the path based on state
+  // if (!state.isOnLandingPage && state.project.id) {
+  //   newURI.setPath(newURI.getPath() + state.project.id + '/');
+  //   newURI.addQueryData('baz', state.mode);
+  //   newURI.addQueryData('bat', state.selection.activePageID);
+  // } else {
+  //   newURI.setPath(newURI.getPath() + state.landingSection + '/');
+  // }
+  //
+  // // Avoid URL thrashing by replacing state while loading instead of pushing
+  // const newPath = newURI.toString();
+  // const oldPath = oldURI.toString();
+  // if (newPath !== oldPath) {
+  //   if (
+  //     (oldState.mode === 'asdf' &&
+  //       state.mode === 'asdf' &&
+  //       !oldState.isOnLandingPage) ||
+  //     oldState.isLoadingProject !== state.isLoadingProject
+  //   ) {
+  //     window.history.replaceState(null, null, newPath);
+  //   } else {
+  //     window.history.pushState(null, null, newPath);
+  //   }
+  // }
+
+  return nextAction;
+};
+
+export default updateURLMiddleware;

client/src/reducers/cascade.js

@@ -1,44 +0,0 @@
-export default function cascadeReducers(arg) {
-  /*
-  Combined a set of cascading reducers into a single reducer.  Cascading
-  reducers are reducers which may rely on state computed by another reducer.
-  Therefore, they:
-  - must be composed in a particular order (currently, this is a simple
-    linear list of reducers, run in list order)
-  - must have access to partially updated "next state" so they can further
-    derive state.
-
-  Parameter is one of:
-  - a Map object
-  - an array of tuples, [ [key1, reducer1], [key2, reducer2], ... ]
-  Ie,  cascadeReducers([ ["a", reduceA], ["b", reduceB] ])
-
-  Each reducer will be called with the sigature:
-      (prevState, action, sharedNextState, sharedPrevState) => newState
-
-  cascadeReducers will build a composite newState object, much
-  like combinedReducers.  Additional semantics:
-  - reducers guaranteed to be called in order
-  - each reducer will receive shared objects
-  */
-  const reducers = arg instanceof Map ? arg : new Map(arg);
-  const reducerKeys = [...reducers.keys()];
-  return (prevState, action) => {
-    const nextState = {};
-    let stateChange = false;
-    for (let i = 0, l = reducerKeys.length; i < l; i += 1) {
-      const key = reducerKeys[i];
-      const reducer = reducers.get(key);
-      const prevStateForKey = prevState ? prevState[key] : undefined;
-      const nextStateForKey = reducer(
-        prevStateForKey,
-        action,
-        nextState,
-        prevState
-      );
-      nextState[key] = nextStateForKey;
-      stateChange = stateChange || nextStateForKey !== prevStateForKey;
-    }
-    return stateChange ? nextState : prevState;
-  };
-}

client/src/reducers/categoricalSelection.js

@@ -1,103 +0,0 @@
-import { ControlsHelpers } from "../util/stateManager";
-import * as globals from "../globals";
-
-function maxCategoryItems(state) {
-  return (
-    state.config.parameters?.["max-category-items"] ??
-    globals.configDefaults.parameters["max-category-items"]
-  );
-}
-
-const CategoricalSelection = (
-  state,
-  action,
-  nextSharedState,
-  prevSharedState
-) => {
-  switch (action.type) {
-    case "initial data load complete (universe exists)":
-    case "set World to current selection":
-    case "reset World to eq Universe":
-    case "set clip quantiles": {
-      const { world } = nextSharedState;
-      return ControlsHelpers.createCategoricalSelection(
-        maxCategoryItems(prevSharedState),
-        world
-      );
-    }
-
-    case "categorical metadata filter select": {
-      /*
-      Set the specific category in this field to false
-      */
-      const newCategorySelected = Array.from(
-        state[action.metadataField].categorySelected
-      );
-      newCategorySelected[action.categoryIndex] = true;
-      const newCategoricalSelection = {
-        ...state,
-        [action.metadataField]: {
-          ...state[action.metadataField],
-          categorySelected: newCategorySelected
-        }
-      };
-      return newCategoricalSelection;
-    }
-
-    case "categorical metadata filter deselect": {
-      /*
-      Set the specific category in this field to false
-      */
-      const newCategorySelected = Array.from(
-        state[action.metadataField].categorySelected
-      );
-      newCategorySelected[action.categoryIndex] = false;
-      const newCategoricalSelection = {
-        ...state,
-        [action.metadataField]: {
-          ...state[action.metadataField],
-          categorySelected: newCategorySelected
-        }
-      };
-      return newCategoricalSelection;
-    }
-
-    case "categorical metadata filter none of these": {
-      /*
-      set all categories in this field to false.
-      */
-      const newCategoricalSelection = {
-        ...state,
-        [action.metadataField]: {
-          ...state[action.metadataField],
-          categorySelected: Array.from(
-            state[action.metadataField].categorySelected
-          ).fill(false)
-        }
-      };
-      return newCategoricalSelection;
-    }
-
-    case "categorical metadata filter all of these": {
-      /*
-      set all categories in this field to true.
-      */
-      const newCategoricalSelection = {
-        ...state,
-        [action.metadataField]: {
-          ...state[action.metadataField],
-          categorySelected: Array.from(
-            state[action.metadataField].categorySelected
-          ).fill(true)
-        }
-      };
-      return newCategoricalSelection;
-    }
-
-    default: {
-      return state;
-    }
-  }
-};
-
-export default CategoricalSelection;

client/src/reducers/colors.js

@@ -1,100 +0,0 @@
-import { createColors } from "../util/stateManager";
-
-const ColorsReducer = (
-  state = {
-    colorMode: null,
-    colorAccessor: null,
-    rgb: null,
-    scale: null
-  },
-  action,
-  nextSharedState,
-  prevSharedState
-) => {
-  switch (action.type) {
-    case "initial data load complete (universe exists)":
-    case "reset World to eq Universe": {
-      const { world } = nextSharedState;
-      const colorMode = null;
-      const colorAccessor = null;
-      const { rgb, scale } = createColors(world, colorMode);
-      return {
-        ...state,
-        colorAccessor,
-        colorMode,
-        rgb,
-        scale
-      };
-    }
-
-    case "set clip quantiles":
-    case "set World to current selection": {
-      const { colorMode, colorAccessor } = state;
-      const { world } = nextSharedState;
-      const { rgb, scale } = createColors(world, colorMode, colorAccessor);
-      return {
-        ...state,
-        rgb,
-        scale
-      };
-    }
-
-    case "reset colorscale": {
-      const { world } = prevSharedState;
-      const { rgb, scale } = createColors(world);
-      return {
-        ...state,
-        colorMode: null,
-        colorAccessor: null,
-        rgb,
-        scale
-      };
-    }
-
-    case "color by categorical metadata":
-    case "color by continuous metadata": {
-      const { world } = prevSharedState;
-
-      /* toggle between this mode and reset */
-      const resetCurrent =
-        action.type === state.colorMode &&
-        action.colorAccessor === state.colorAccessor;
-      const colorMode = !resetCurrent ? action.type : null;
-      const colorAccessor = !resetCurrent ? action.colorAccessor : null;
-
-      const { rgb, scale } = createColors(world, colorMode, colorAccessor);
-      return {
-        ...state,
-        colorMode,
-        colorAccessor,
-        rgb,
-        scale
-      };
-    }
-
-    case "color by expression": {
-      const { world } = prevSharedState;
-
-      /* toggle between this mode and reset */
-      const resetCurrent =
-        action.type === state.colorMode && action.gene === state.colorAccessor;
-      const colorMode = !resetCurrent ? action.type : null;
-      const colorAccessor = !resetCurrent ? action.gene : null;
-
-      const { rgb, scale } = createColors(world, colorMode, colorAccessor);
-      return {
-        ...state,
-        colorMode,
-        colorAccessor,
-        rgb,
-        scale
-      };
-    }
-
-    default: {
-      return state;
-    }
-  }
-};
-
-export default ColorsReducer;

client/src/reducers/continuousSelection.js

@@ -1,35 +0,0 @@
-import { makeContinuousDimensionName } from "../util/nameCreators";
-
-const ContinuousSelection = (state = {}, action) => {
-  switch (action.type) {
-    case "reset World to eq Universe":
-    case "set clip quantiles": {
-      return {};
-    }
-    case "continuous metadata histogram start":
-    case "continuous metadata histogram brush":
-    case "continuous metadata histogram end": {
-      const name = makeContinuousDimensionName(
-        action.continuousNamespace,
-        action.selection
-      );
-      return {
-        ...state,
-        [name]: action.range
-      };
-    }
-    case "continuous metadata histogram cancel": {
-      const name = makeContinuousDimensionName(
-        action.continuousNamespace,
-        action.selection
-      );
-      const { [name]: deletedField, ...newState } = state;
-      return newState;
-    }
-    default: {
-      return state;
-    }
-  }
-};
-
-export default ContinuousSelection;

client/src/reducers/controls.js

@@ -1,8 +1,102 @@
 // jshint esversion: 6
 
 import _ from "lodash";
+import { World, kvCache } from "../util/stateManager";
+import parseRGB from "../util/parseRGB";
+import Crossfilter from "../util/typedCrossfilter";
+import * as globals from "../globals";
+import {
+  layoutDimensionName,
+  obsAnnoDimensionName,
+  userDefinedDimensionName,
+  diffexpDimensionName,
+  makeContinuousDimensionName
+} from "../util/nameCreators";
+import { fillRange } from "../util/typedCrossfilter/util";
 
-import { WorldUtil } from "../util/stateManager";
+/*
+Selection state for categoricals are tracked in an Object that
+has two main components for each category:
+1. mapping of option value to an index
+2. array of bool selection state by index
+Remember that option values can be ANY js type, except undefined/null.
+
+  {
+    _category_name_1: {
+      // map of option value to index
+      optionIndex: Map([
+        optval1: index,
+        ...
+      ])
+
+      // index->selection true/false state
+      optionSelected: [ true/false, true/false, ... ]
+
+      // number of options
+      numOptions: number,
+
+      // isTruncated - true if the options for selection has
+      // been truncated (ie, was too large to implement)
+    }
+  }
+*/
+function topNoptions(summary) {
+  const counts = _.map(summary.categories, cat => summary.options[cat]);
+  const sortIndex = fillRange(new Array(summary.numOptions)).sort(
+    (a, b) => counts[b] - counts[a]
+  );
+  const sortedCategories = _.map(sortIndex, i => summary.categories[i]);
+  const sortedCounts = _.map(sortIndex, i => counts[i]);
+  const N = globals.maxCategoricalOptionsToDisplay;
+
+  if (sortedCategories.length < N) {
+    return [sortedCategories, sortedCounts];
+  }
+  return [sortedCategories.slice(0, N), sortedCounts.slice(0, N)];
+}
+
+function createCategoricalSelectionState(state, world) {
+  const res = {};
+  _.forEach(world.summary.obs, (value, key) => {
+    if (value.categories) {
+      const isColorField = key.includes("color") || key.includes("Color");
+      const isSelectableCategory =
+        !isColorField &&
+        key !== "name" &&
+        value.categories.length < state.maxCategoryItems;
+      if (isSelectableCategory) {
+        const [optionValue, optionCount] = topNoptions(value);
+        // const optionCount = Object.values(value.options);
+
+        const optionIndex = new Map(optionValue.map((v, i) => [v, i]));
+        const numOptions = optionIndex.size;
+        const optionSelected = new Array(numOptions).fill(true);
+        const isTruncated = optionValue.length < value.numOptions;
+        res[key] = {
+          optionValue, // array: of natively typed option values
+          optionIndex, // map: option value (native type) -> option index
+          optionSelected, // array: t/f selection state
+          numOptions, // number: of options
+          isTruncated, // bool: true if list was truncated
+          optionCount // array: cardinality of each option
+        };
+      }
+    }
+  });
+  return res;
+}
+
+/*
+given a categoricalSelectionState, return the list of all option values
+where selection state is true (ie, they are selected).
+*/
+function selectedValuesForCategory(categorySelectionState) {
+  const selectedValues = _([...categorySelectionState.optionIndex])
+    .filter(tuple => categorySelectionState.optionSelected[tuple[1]])
+    .map(tuple => tuple[0])
+    .value();
+  return selectedValues;
+}
 
 const Controls = (
   state = {
@@ -10,23 +104,36 @@ const Controls = (
     loading: false,
     error: null,
 
+    // configuration
+    maxCategoryItems: globals.configDefaults.parameters["max-category-items"],
+
+    // the whole big bang
+    universe: null,
+
     // all of the data + selection state
+    world: null,
+    colorName: null,
+    colorRGB: null,
+    categoricalSelectionState: null,
+    crossfilter: null,
+    dimensionMap: null,
     userDefinedGenes: [],
-    userDefinedGenesLoading: false,
     diffexpGenes: [],
 
-    resettingInterface: false,
+    colorAccessor: null,
+    colorScale: null,
 
     opacityForDeselectedCells: 0.2,
+    graphBrushSelection: null,
+    continuousSelection: null,
     scatterplotXXaccessor: null, // just easier to read
     scatterplotYYaccessor: null,
+    axesHaveBeenDrawn: false,
     graphRenderCounter: 0 /* integer as <Component key={graphRenderCounter} - a change in key forces a remount */,
     __storedStateForCelllist1__: null /* will need procedural control of brush ie., brush.extent https://bl.ocks.org/micahstubbs/3cda05ca68cba260cb81 */,
     __storedStateForCelllist2__: null
   },
-  action,
+  action
-  nextSharedState,
-  prevSharedState
 ) => {
   /*
   For now, log anything looking like an error to the console.
@@ -40,91 +147,308 @@ const Controls = (
           Initialization, World/Universe management
           and data loading.
     ******************************************************/
+    case "configuration load complete": {
+      // there are a couple of configuration items we need to retain
+      return {
+        ...state,
+        maxCategoryItems: _.get(
+          state.config,
+          "parameters.max-category-items",
+          globals.configDefaults.parameters["max-category-items"]
+        )
+      };
+    }
     case "initial data load start": {
       return { ...state, loading: true };
     }
-    case "initial data load complete (universe exists)": {
+    case "initial data load complete (universe exists)":
+    case "reset World to eq Universe": {
+      const { userDefinedGenes, diffexpGenes } = state;
       /* first light - create world & other data-driven defaults */
-      WorldUtil.clearCaches();
+      const { universe } = action;
+      const world = World.createWorldFromEntireUniverse(universe);
+      const colorName = new Array(universe.nObs).fill(globals.defaultCellColor);
+      const colorRGB = _.map(colorName, c => parseRGB(c));
+      const categoricalSelectionState = createCategoricalSelectionState(
+        state,
+        world
+      );
+      const crossfilter = Crossfilter(world.obsAnnotations);
+      const dimensionMap = World.createObsDimensionMap(crossfilter, world);
+
+      const worldVarDataCache = world.varDataCache;
+
+      // dimensionMap = {
+      //     layout_X: dim-for-X,
+      //     obsAnno_name: dim for an annotation,
+      //     varData_userDefined_genename: dim for user defined expression,
+      //     varData_diffexp_genename: dim for diff-exp added gene expression
+      // }
+      /* var dimensions */
+      if (userDefinedGenes.length > 0) {
+        /*
+          verbose & slightly confusing that we also access this as an object
+          in controls rather than an array, should be abstracted into
+          util ie., createDimensionsFromBothListsOfGenes(userGenes, diffExp)
+        */
+        _.forEach(userDefinedGenes, gene => {
+          dimensionMap[
+            userDefinedDimensionName(gene)
+          ] = World.createVarDimension(
+            /* "__var__" + */
+            world,
+            worldVarDataCache,
+            crossfilter,
+            gene
+          );
+        });
+      }
+
+      if (diffexpGenes.length > 0) {
+        _.forEach(diffexpGenes, gene => {
+          dimensionMap[diffexpDimensionName(gene)] = World.createVarDimension(
+            /* "__var__" + */
+            world,
+            worldVarDataCache,
+            crossfilter,
+            gene
+          );
+        });
+      }
+
       return {
         ...state,
         loading: false,
         error: null,
-        resettingInterface: false
+        universe,
-      };
+        world,
-    }
+        colorName,
-    case "reset World to eq Universe": {
+        colorRGB,
-      WorldUtil.clearCaches();
+        categoricalSelectionState,
-      return {
+        crossfilter,
-        ...state,
+        dimensionMap,
-        resettingInterface: false
+        colorAccessor: null
       };
     }
     case "set World to current selection": {
-      WorldUtil.clearCaches();
+      const { userDefinedGenes, diffexpGenes } = state;
+
+      /* Set viewable world to be the currently selected data */
+      const world = World.createWorldFromCurrentSelection(
+        action.universe,
+        action.world,
+        action.crossfilter
+      );
+      const colorName = new Array(world.nObs).fill(globals.defaultCellColor);
+      const colorRGB = _.map(colorName, c => parseRGB(c));
+      const categoricalSelectionState = createCategoricalSelectionState(
+        state,
+        world
+      );
+      const crossfilter = Crossfilter(world.obsAnnotations);
+      const dimensionMap = World.createObsDimensionMap(crossfilter, world);
+
+      const worldVarDataCache = world.varDataCache;
+      /* var dimensions */
+
+      if (userDefinedGenes.length > 0) {
+        /*
+          verbose & slightly confusing that we also access this as an object
+          in controls rather than an array, should be abstracted into
+          util ie., createDimensionsFromBothListsOfGenes(userGenes, diffExp)
+        */
+        _.forEach(userDefinedGenes, gene => {
+          dimensionMap[
+            userDefinedDimensionName(gene)
+          ] = World.createVarDimension(
+            /* "__var__" + */
+            world,
+            worldVarDataCache,
+            crossfilter,
+            gene
+          );
+        });
+      }
+
+      if (diffexpGenes.length > 0) {
+        _.forEach(diffexpGenes, gene => {
+          dimensionMap[diffexpDimensionName(gene)] = World.createVarDimension(
+            /* "__var__" + */
+            world,
+            worldVarDataCache,
+            crossfilter,
+            gene
+          );
+        });
+      }
+
       return {
         ...state,
         loading: false,
-        error: null
+        error: null,
+        world,
+        colorName,
+        colorRGB,
+        categoricalSelectionState,
+        crossfilter,
+        dimensionMap,
+        colorAccessor: null
       };
     }
-    case "request user defined gene started": {
+    case "expression load success": {
+      const { world, universe } = state;
+      let universeVarDataCache = universe.varDataCache;
+      let worldVarDataCache = world.varDataCache;
+      _.forEach(action.expressionData, (val, key) => {
+        universeVarDataCache = kvCache.set(universeVarDataCache, key, val);
+        if (kvCache.get(worldVarDataCache, key) === undefined) {
+          worldVarDataCache = kvCache.set(
+            worldVarDataCache,
+            key,
+            World.subsetVarData(world, universe, val)
+          );
+        }
+      });
+
       return {
         ...state,
-        userDefinedGenesLoading: true
+        universe: {
-      };
+          ...universe,
-    }
+          varDataCache: universeVarDataCache
-    case "request user defined gene error": {
+        },
-      return {
+        world: {
-        ...state,
+          ...world,
-        userDefinedGenesLoading: false
+          varDataCache: worldVarDataCache
+        }
       };
     }
     case "request user defined gene success": {
-      const { userDefinedGenes } = state;
+      const { world, crossfilter, dimensionMap, userDefinedGenes } = state;
-      const _userDefinedGenes = _.uniq(
+      const worldVarDataCache = world.varDataCache;
-        userDefinedGenes.concat(action.data.genes)
+      const _userDefinedGenes = userDefinedGenes.slice();
+      const gene = action.data.genes[0];
+
+      dimensionMap[userDefinedDimensionName(gene)] = World.createVarDimension(
+        /* "__var__" + */
+        world,
+        worldVarDataCache,
+        crossfilter,
+        gene
       );
+
       return {
         ...state,
-        userDefinedGenes: _userDefinedGenes,
+        dimensionMap,
-        userDefinedGenesLoading: false
+        userDefinedGenes: _userDefinedGenes
       };
     }
     case "request differential expression success": {
-      const { world } = prevSharedState;
+      const { world, crossfilter, dimensionMap } = state;
+      const worldVarDataCache = world.varDataCache;
       const _diffexpGenes = [];
+
       action.data.forEach(d => {
-        _diffexpGenes.push(world.varAnnotations.at(d[0], "name"));
+        _diffexpGenes.push(world.varAnnotations[d[0]].name);
       });
+
+      _.forEach(_diffexpGenes, gene => {
+        dimensionMap[diffexpDimensionName(gene)] = World.createVarDimension(
+          /* "__var__" + */
+          world,
+          worldVarDataCache,
+          crossfilter,
+          gene
+        );
+      });
+
       return {
         ...state,
+        dimensionMap,
         diffexpGenes: _diffexpGenes
       };
     }
     case "clear differential expression": {
+      const { world, universe, dimensionMap } = state;
+      const _dimensionMap = dimensionMap;
+      const universeVarDataCache = universe.varDataCache;
+      const worldVarDataCache = world.varDataCache;
+
+      _.forEach(action.diffExp, values => {
+        const { name } = world.varAnnotations[values[0]];
+        // clean up crossfilter dimensions
+        const dimension = dimensionMap[diffexpDimensionName(name)];
+        dimension.dispose();
+        delete dimensionMap[diffexpDimensionName(name)];
+      });
       return {
         ...state,
-        diffexpGenes: []
+        dimensionMap: _dimensionMap,
+        diffexpGenes: [],
+        universe: {
+          ...universe,
+          varDataCache: universeVarDataCache
+        },
+        world: {
+          ...world,
+          varDataCache: worldVarDataCache
+        }
       };
     }
-    case "clear user defined gene": {
+    case "user defined gene": {
-      const { userDefinedGenes } = state;
+      /*
-      const newUserDefinedGenes = _.filter(
+        this could also live in expression success with a conditional,
-        userDefinedGenes,
+        but that handles diffexp also
-        d => d !== action.data
+      */
-      );
+      const newUserDefinedGenes = state.userDefinedGenes.slice();
+      newUserDefinedGenes.push(action.data);
       return {
         ...state,
         userDefinedGenes: newUserDefinedGenes
       };
     }
-    case "clear all user defined genes": {
+    case "clear user defined gene": {
+      const { userDefinedGenes, dimensionMap } = state;
+      const newUserDefinedGenes = _.filter(
+        userDefinedGenes,
+        d => d !== action.data
+      );
+
+      const dimension = dimensionMap[userDefinedDimensionName(action.data)];
+      dimension.dispose();
+      delete dimensionMap[userDefinedDimensionName(action.data)];
+
       return {
         ...state,
+        dimensionMap,
+        userDefinedGenes: newUserDefinedGenes
+      };
+    }
+    case "clear all user defined genes": {
+      const { userDefinedGenes, dimensionMap } = state;
+
+      _.forEach(userDefinedGenes, gene => {
+        const dimension = dimensionMap[userDefinedDimensionName(gene)];
+        dimension.dispose();
+        delete dimensionMap[userDefinedDimensionName(gene)];
+      });
+
+      return {
+        ...state,
+        dimensionMap,
         userDefinedGenes: []
       };
     }
+    case "reset colorscale": {
+      const { world } = state;
+      const colorName = new Array(world.nObs).fill(globals.defaultCellColor);
+      const colorRGB = _.map(colorName, c => parseRGB(c));
+      return {
+        ...state,
+        colorName,
+        colorRGB,
+        colorAccessor: null
+      };
+    }
     case "expression load error":
     case "initial data load error": {
       return {
@@ -137,6 +461,43 @@ const Controls = (
     /*******************************
              User Events
      *******************************/
+    case "graph brush selection change": {
+      state.dimensionMap[layoutDimensionName("X")].filterRange([
+        action.brushCoords.northwest[0],
+        action.brushCoords.southeast[0]
+      ]);
+      state.dimensionMap[layoutDimensionName("Y")].filterRange([
+        action.brushCoords.southeast[1],
+        action.brushCoords.northwest[1]
+      ]);
+      return {
+        ...state,
+        graphBrushSelection: action.brushCoords
+      };
+    }
+    case "graph brush deselect": {
+      state.dimensionMap[layoutDimensionName("X")].filterAll();
+      state.dimensionMap[layoutDimensionName("Y")].filterAll();
+      return {
+        ...state,
+        graphBrushSelection: null
+      };
+    }
+    case "continuous metadata histogram brush": {
+      const name = makeContinuousDimensionName(
+        action.continuousNamespace,
+        action.selection
+      );
+
+      // action.selection: metadata name being selected
+      // action.range: filter range, or null if deselected
+      if (!action.range) {
+        state.dimensionMap[name].filterAll();
+      } else {
+        state.dimensionMap[name].filterRange(action.range);
+      }
+      return { ...state };
+    }
     case "change opacity deselected cells in 2d graph background":
       return {
         ...state,
@@ -149,10 +510,114 @@ const Controls = (
         graphRenderCounter: c
       };
     }
-    case "interface reset started": {
+    /*******************************
+          Categorical metadata
+    *******************************/
+    case "categorical metadata filter select": {
+      const newOptionSelected = Array.from(
+        state.categoricalSelectionState[action.metadataField].optionSelected
+      );
+      newOptionSelected[action.optionIndex] = true;
+      const newCategoricalSelectionState = {
+        ...state.categoricalSelectionState,
+        [action.metadataField]: {
+          ...state.categoricalSelectionState[action.metadataField],
+          optionSelected: newOptionSelected
+        }
+      };
+
+      // update the filter to match all selected options
+      const cat = newCategoricalSelectionState[action.metadataField];
+      state.dimensionMap[obsAnnoDimensionName(action.metadataField)].filterEnum(
+        selectedValuesForCategory(cat)
+      );
+
       return {
         ...state,
-        resettingInterface: true
+        categoricalSelectionState: newCategoricalSelectionState
+      };
+    }
+    case "categorical metadata filter deselect": {
+      const newOptionSelected = Array.from(
+        state.categoricalSelectionState[action.metadataField].optionSelected
+      );
+      newOptionSelected[action.optionIndex] = false;
+      const newCategoricalSelectionState = {
+        ...state.categoricalSelectionState,
+        [action.metadataField]: {
+          ...state.categoricalSelectionState[action.metadataField],
+          optionSelected: newOptionSelected
+        }
+      };
+
+      // update the filter to match all selected options
+      const cat = newCategoricalSelectionState[action.metadataField];
+      state.dimensionMap[obsAnnoDimensionName(action.metadataField)].filterEnum(
+        selectedValuesForCategory(cat)
+      );
+
+      return {
+        ...state,
+        categoricalSelectionState: newCategoricalSelectionState
+      };
+    }
+    case "categorical metadata filter none of these": {
+      const newCategoricalSelectionState = {
+        ...state.categoricalSelectionState,
+        [action.metadataField]: {
+          ...state.categoricalSelectionState[action.metadataField],
+          optionSelected: Array.from(
+            state.categoricalSelectionState[action.metadataField].optionSelected
+          ).fill(false)
+        }
+      };
+      state.dimensionMap[
+        obsAnnoDimensionName(action.metadataField)
+      ].filterNone();
+      return {
+        ...state,
+        categoricalSelectionState: newCategoricalSelectionState
+      };
+    }
+    case "categorical metadata filter all of these": {
+      const newCategoricalSelectionState = {
+        ...state.categoricalSelectionState,
+        [action.metadataField]: {
+          ...state.categoricalSelectionState[action.metadataField],
+          optionSelected: Array.from(
+            state.categoricalSelectionState[action.metadataField].optionSelected
+          ).fill(true)
+        }
+      };
+      state.dimensionMap[
+        obsAnnoDimensionName(action.metadataField)
+      ].filterAll();
+      return {
+        ...state,
+        categoricalSelectionState: newCategoricalSelectionState
+      };
+    }
+
+    /*******************************
+              Color Scale
+    *******************************/
+    case "color by categorical metadata":
+    case "color by continuous metadata": {
+      return {
+        ...state,
+        colorName: action.colors.name,
+        colorRGB: action.colors.rgb,
+        colorAccessor: action.colorAccessor,
+        colorScale: action.colorScale
+      };
+    }
+    case "color by expression": {
+      return {
+        ...state,
+        colorName: action.colors.name,
+        colorRGB: action.colors.rgb,
+        colorAccessor: action.gene,
+        colorScale: action.colorScale
       };
     }
 

client/src/reducers/crossfilter.js

@@ -1,189 +0,0 @@
-import _ from "lodash";
-
-import Crossfilter from "../util/typedCrossfilter";
-import { World, ControlsHelpers } from "../util/stateManager";
-import {
-  layoutDimensionName,
-  obsAnnoDimensionName,
-  userDefinedDimensionName,
-  diffexpDimensionName,
-  makeContinuousDimensionName
-} from "../util/nameCreators";
-
-const XYDimName = layoutDimensionName("XY");
-
-const CrossfilterReducer = (
-  state = null,
-  action,
-  nextSharedState,
-  prevSharedState
-) => {
-  switch (action.type) {
-    case "initial data load complete (universe exists)": {
-      const { world } = nextSharedState;
-      const crossfilter = World.createObsDimensions(
-        new Crossfilter(world.obsAnnotations),
-        world
-      );
-      return crossfilter;
-    }
-
-    case "reset World to eq Universe": {
-      const { userDefinedGenes, diffexpGenes } = prevSharedState.controls;
-      const { world } = nextSharedState;
-      const crossfilter = ControlsHelpers.createGeneDimensions(
-        userDefinedGenes,
-        diffexpGenes,
-        world,
-        prevSharedState.resetCache.crossfilter
-      );
-      return crossfilter;
-    }
-
-    case "set clip quantiles":
-    case "set World to current selection": {
-      const { userDefinedGenes, diffexpGenes } = prevSharedState.controls;
-      const { world } = nextSharedState;
-      let crossfilter = new Crossfilter(world.obsAnnotations);
-      crossfilter = World.createObsDimensions(crossfilter, world);
-      crossfilter = ControlsHelpers.createGeneDimensions(
-        userDefinedGenes,
-        diffexpGenes,
-        world,
-        crossfilter
-      );
-      return crossfilter;
-    }
-
-    case "request user defined gene success": {
-      const { world } = prevSharedState;
-      const gene = action.data.genes[0];
-      return state.addDimension(
-        userDefinedDimensionName(gene),
-        "scalar",
-        world.varData.col(gene).asArray(),
-        Float32Array
-      );
-    }
-
-    case "request differential expression success": {
-      const { world } = prevSharedState;
-      const genes = _.map(action.data, d =>
-        world.varAnnotations.at(d[0], "name")
-      );
-      const crossfilter = _.reduce(
-        genes,
-        (xfltr, gene) =>
-          xfltr.addDimension(
-            diffexpDimensionName(gene),
-            "scalar",
-            world.varData.col(gene).asArray(),
-            Float32Array
-          ),
-        state
-      );
-      return crossfilter;
-    }
-
-    case "clear differential expression": {
-      const { world } = prevSharedState;
-      const crossfilter = _.reduce(
-        action.diffExp,
-        (xfltr, values) => {
-          const name = world.varAnnotations.at(values[0], "name");
-          return xfltr.delDimension(diffexpDimensionName(name));
-        },
-        state
-      );
-      return crossfilter;
-    }
-
-    case "clear user defined gene": {
-      return state.delDimension(userDefinedDimensionName(action.data));
-    }
-
-    case "clear all user defined genes": {
-      const { userDefinedGenes } = prevSharedState.controls;
-      const crossfilter = _.reduce(
-        userDefinedGenes,
-        (xfltr, gene) => xfltr.delDimension(userDefinedDimensionName(gene)),
-        state
-      );
-      return crossfilter;
-    }
-
-    case "graph brush end":
-    case "graph brush change": {
-      const [minX, maxY] = action.brushCoords.northwest;
-      const [maxX, minY] = action.brushCoords.southeast;
-      return state.select(XYDimName, {
-        mode: "within-rect",
-        minX,
-        minY,
-        maxX,
-        maxY
-      });
-    }
-
-    case "graph lasso end": {
-      const { polygon } = action;
-      return state.select(XYDimName, {
-        mode: "within-polygon",
-        polygon
-      });
-    }
-
-    case "graph lasso cancel":
-    case "graph brush cancel":
-    case "graph lasso deselect":
-    case "graph brush deselect": {
-      return state.select(XYDimName, { mode: "all" });
-    }
-
-    case "continuous metadata histogram start":
-    case "continuous metadata histogram brush":
-    case "continuous metadata histogram cancel":
-    case "continuous metadata histogram end": {
-      const name = makeContinuousDimensionName(
-        action.continuousNamespace,
-        action.selection
-      );
-      // action.selection: metadata name being selected
-      // action.range: filter range, or null if deselected
-      if (!action.range) {
-        return state.select(name, { mode: "all" });
-      }
-      const [lo, hi] = action.range;
-      const newState = state.select(name, { mode: "range", lo, hi });
-      return newState;
-    }
-
-    case "categorical metadata filter select":
-    case "categorical metadata filter deselect": {
-      const { categoricalSelection } = nextSharedState;
-      const cat = categoricalSelection[action.metadataField];
-      return state.select(obsAnnoDimensionName(action.metadataField), {
-        mode: "exact",
-        values: ControlsHelpers.selectedValuesForCategory(cat)
-      });
-    }
-
-    case "categorical metadata filter none of these": {
-      return state.select(obsAnnoDimensionName(action.metadataField), {
-        mode: "none"
-      });
-    }
-
-    case "categorical metadata filter all of these": {
-      return state.select(obsAnnoDimensionName(action.metadataField), {
-        mode: "all"
-      });
-    }
-
-    default: {
-      return state;
-    }
-  }
-};
-
-export default CrossfilterReducer;

client/src/reducers/graphSelection.js

@@ -1,60 +0,0 @@
-const GraphSelection = (
-  state = {
-    tool: "lasso", // what selection tool mode (lasso, brush, ...)
-    selection: { mode: "all" } // current selection, which is tool specific
-  },
-  action
-) => {
-  switch (action.type) {
-    case "set clip quantiles":
-    case "reset World to eq Universe": {
-      return {
-        ...state,
-        selection: {
-          mode: "all"
-        }
-      };
-    }
-
-    case "graph brush end":
-    case "graph brush change": {
-      const { brushCoords } = action;
-      return {
-        ...state,
-        selection: {
-          mode: "within-rect",
-          brushCoords
-        }
-      };
-    }
-
-    case "graph lasso end": {
-      const { polygon } = action;
-      return {
-        ...state,
-        selection: {
-          mode: "within-polygon",
-          polygon
-        }
-      };
-    }
-
-    case "graph lasso cancel":
-    case "graph brush cancel":
-    case "graph lasso deselect":
-    case "graph brush deselect": {
-      return {
-        ...state,
-        selection: {
-          mode: "all"
-        }
-      };
-    }
-
-    default: {
-      return state;
-    }
-  }
-};
-
-export default GraphSelection;

client/src/reducers/index.js

@@ -1,51 +1,27 @@
-import { createStore, applyMiddleware } from "redux";
+// jshint esversion: 6
+import { combineReducers, createStore, applyMiddleware } from "redux";
 import thunk from "redux-thunk";
+import updateURLMiddleware from "../middleware/updateURLMiddleware";
+import updateCellColors from "../middleware/updateCellColors";
+import { composeWithDevTools } from "redux-devtools-extension";
 
-import cascadeReducers from "./cascade";
-import undoable from "./undoable";
 import config from "./config";
-import universe from "./universe";
-import world from "./world";
-import categoricalSelection from "./categoricalSelection";
-import continuousSelection from "./continuousSelection";
-import graphSelection from "./graphSelection";
-import crossfilter from "./crossfilter";
-import colors from "./colors";
 import differential from "./differential";
 import responsive from "./responsive";
 import controls from "./controls";
-import resetCache from "./resetCache";
 
-import undoableConfig from "./undoableConfig";
+const Reducer = combineReducers({
+  config,
+  responsive,
+  controls,
+  differential
+});
 
-const Reducer = undoable(
+const store = createStore(
-  cascadeReducers([
+  Reducer,
-    ["config", config],
+  composeWithDevTools(
-    ["universe", universe],
+    applyMiddleware(thunk, updateURLMiddleware, updateCellColors)
-    ["world", world],
+  )
-    ["categoricalSelection", categoricalSelection],
-    ["continuousSelection", continuousSelection],
-    ["graphSelection", graphSelection],
-    ["crossfilter", crossfilter],
-    ["colors", colors],
-    ["controls", controls],
-    ["differential", differential],
-    ["responsive", responsive],
-    ["resetCache", resetCache]
-  ]),
-  [
-    "world",
-    "categoricalSelection",
-    "continuousSelection",
-    "graphSelection",
-    "crossfilter",
-    "colors",
-    "controls",
-    "differential"
-  ],
-  undoableConfig
 );
 
-const store = createStore(Reducer, applyMiddleware(thunk));
-
 export default store;

client/src/reducers/resetCache.js

@@ -1,31 +0,0 @@
-/*
-Reducer which caches derived state to be used in a reset or other 
-recomputation.
-
-Currently this only caches the baseline (full universe) world & crossfilter, 
-for use in a Reset.
-*/
-const ResetCacheReducer = (
-  state = {
-    world: null,
-    crossfilter: null
-  },
-  action,
-  nextSharedState
-) => {
-  switch (action.type) {
-    case "initial data load complete (universe exists)": {
-      const { world, crossfilter } = nextSharedState;
-      return {
-        ...state,
-        world,
-        crossfilter
-      };
-    }
-    default: {
-      return state;
-    }
-  }
-};
-
-export default ResetCacheReducer;

client/src/reducers/undoable.js

@@ -1,287 +0,0 @@
-/*
-A redo/undo meta reducer for Redux.   Designed to work well with the cascadeReducer().
-
-Requires three parameters:
-* reducer - a reducer, which MUST return an object as state.
-* undoableKeys - an array of object keys (strings).   If any of these keys
-  are in the object/state returned by the reducer, they will be treated as
-  state to be made "undoable".
-* options - an optional object, which may contain the following parameters:
-    * historyLimit: max number of historical states to remember (aka max undo depth)
-    * actionFilter: filter function, (state, action, filterState) => value.
-      See below for details.
-    * debug: if truish, will print helpful log messages about history manipulation
-
-This meta reducer accepts three actions types:
-* @@undoable/undo - move back in history
-* @@undoable/redo - move forward in history
-* @@undoable/clear - clear history
-
----
-
-Action filter - controls the undoable reducer side-effects.  If not
-specified, the action filter defaults to "save", ie, pushes a redo
-point upon each action.
-
-The action filter callback has access to the current action, the entire
-undoable reducer state, and any state it wants to manage ("filterState").
-This filter state will be passed to each action filter call, and any
-value returned (via @@undoable/filterState field described below) will be
-MERGED into the current filter state.
-
-An object must be returned (the "undoable action"), indicating desired
-history state processing.  The undoable action object contents, by key:
-
-  @@undoable/filterAction: required.   Can be one of:
-      "skip" - reduce the current action, but no other side effects.
-        Same as returning false.
-      "clear" - reduce the current action, and clear history state.
-      "save" - push the previous state onto the history stack (ie,
-        before reducing the action)
-      "stashPending" - reduce action, save state as pending.  Does not
-        not commit it to history.  Along with cancelPending and applyPending,
-        can be used to delay commit of history (eg, for multi-action
-        groupings, asynch operations, etc).
-      "cancelPending" - reduce action, cancel any pending state save.
-      "applyPending" - commit any pending state to the history stack,
-        then reduce action.
-
-  @@undoable/filterState: optional.  If this value is set, it will be
-    MERGED into the current filter state. The value and semantics of any
-    filter state are entirely at the discretion of the action filter.
-
-*/
-import fromEntries from "../util/fromEntries";
-
-const historyKeyPrefix = "@@undoable/";
-const pastKey = `${historyKeyPrefix}past`;
-const futureKey = `${historyKeyPrefix}future`;
-const filterStateKey = `${historyKeyPrefix}filterState`;
-const filterActionKey = `${historyKeyPrefix}filterAction`;
-const pendingKey = `${historyKeyPrefix}pending`;
-const defaultHistoryLimit = -100;
-
-const Undoable = (reducer, undoableKeys, options = {}) => {
-  const { debug } = options;
-  let { historyLimit } = options;
-  if (!historyLimit) historyLimit = defaultHistoryLimit;
-  if (historyLimit > 0) historyLimit = -historyLimit;
-  const actionFilter =
-    options.actionFilter || (() => ({ [filterActionKey]: "save" }));
-
-  if (!Array.isArray(undoableKeys) || undoableKeys.length === 0)
-    throw new Error("undoable keys array must be specified");
-  const undoableKeysSet = new Set(undoableKeys);
-
-  /*
-  Undo the current to previous history
-  */
-  function undo(currentState) {
-    const past = currentState[pastKey];
-    const future = currentState[futureKey];
-    if (past.length === 0) return currentState;
-    const currentUndoableState = Object.entries(currentState).filter(kv =>
-      undoableKeysSet.has(kv[0])
-    );
-    const newPast = [...past];
-    const newState = newPast.pop();
-    const newFuture = push(future, currentUndoableState);
-    const nextState = {
-      ...currentState,
-      ...fromEntries(newState),
-      [pastKey]: newPast,
-      [futureKey]: newFuture,
-      [pendingKey]: null
-    };
-    return nextState;
-  }
-
-  /*
-  Replay future, previously undone.
-  */
-  function redo(currentState) {
-    const past = currentState[pastKey] || [];
-    const future = currentState[futureKey] || [];
-    if (future.length === 0) return currentState;
-    const currentUndoableState = Object.entries(currentState).filter(kv =>
-      undoableKeysSet.has(kv[0])
-    );
-    const newFuture = [...future];
-    const newState = newFuture.pop();
-    const newPast = push(past, currentUndoableState);
-    const nextState = {
-      ...currentState,
-      ...fromEntries(newState),
-      [pastKey]: newPast,
-      [futureKey]: newFuture,
-      [pendingKey]: null
-    };
-    return nextState;
-  }
-
-  /*
-  Clear the history state.  No side-effects on current state.
-  */
-  function clear(currentState) {
-    return {
-      ...currentState,
-      [pastKey]: [],
-      [futureKey]: [],
-      [filterStateKey]: {},
-      [pendingKey]: null
-    };
-  }
-
-  /*
-  Reduce current action, with no history side-effects
-  */
-  function skip(currentState, action, filterState) {
-    const past = currentState[pastKey] || [];
-    const pending = currentState[pendingKey];
-    const res = reducer(currentState, action);
-    return {
-      ...res,
-      [pastKey]: past,
-      [futureKey]: [],
-      [filterStateKey]: filterState,
-      [pendingKey]: pending
-    };
-  }
-
-  /*
-  Save current state in the history, then reduce action.
-  */
-  function save(currentState, action, filterState) {
-    const past = currentState[pastKey] || [];
-    const currentUndoableState = Object.entries(currentState).filter(kv =>
-      undoableKeysSet.has(kv[0])
-    );
-    const res = reducer(currentState, action);
-    const newPast = push(past, currentUndoableState, historyLimit);
-    const nextState = {
-      ...res,
-      [pastKey]: newPast,
-      [futureKey]: [],
-      [filterStateKey]: filterState,
-      [pendingKey]: null
-    };
-    return nextState;
-  }
-
-  /*
-  Save current state as pending history change.  No other side effects.
-  */
-  function stashPending(currentState) {
-    const currentUndoableState = Object.entries(currentState).filter(kv =>
-      undoableKeysSet.has(kv[0])
-    );
-    return {
-      ...currentState,
-      [pendingKey]: currentUndoableState
-    };
-  }
-
-  /*
-  Cancel pending history state change.  No other side effects.
-  */
-  function cancelPending(currentState) {
-    return {
-      ...currentState,
-      [pendingKey]: null
-    };
-  }
-
-  /*
-  Push pending state onto the history stack
-  */
-  function applyPending(currentState) {
-    const past = currentState[pastKey] || [];
-    const pendingState = currentState[pendingKey];
-    const newPast = push(past, pendingState, historyLimit);
-    const nextState = {
-      ...currentState,
-      [pastKey]: newPast,
-      [futureKey]: [],
-      [pendingKey]: null
-    };
-    return nextState;
-  }
-
-  return (
-    currentState = {
-      [pastKey]: [],
-      [futureKey]: [],
-      [filterStateKey]: {},
-      [pendingKey]: null
-    },
-    action
-  ) => {
-    const aType = action.type;
-    switch (aType) {
-      case "@@undoable/undo": {
-        return undo(currentState, action);
-      }
-
-      case "@@undoable/redo": {
-        return redo(currentState, action);
-      }
-
-      case "@@undoable/clear": {
-        return clear(currentState, action);
-      }
-
-      default: {
-        const currentFilterState = currentState[filterStateKey];
-        const actionFilterResp = actionFilter(
-          currentState,
-          action,
-          currentFilterState
-        );
-        const {
-          [filterActionKey]: filterAction,
-          [filterStateKey]: filterStateUpdate
-        } = actionFilterResp;
-        const nextFilterState = { ...currentFilterState, ...filterStateUpdate };
-
-        switch (filterAction) {
-          case "clear":
-            if (debug) console.log("---- CLEAR HISTO", action.type);
-            return clear(skip(currentState, action, nextFilterState));
-
-          case "save":
-            if (debug) console.log("---- SAVE HISTO", action.type);
-            return save(currentState, action, nextFilterState);
-
-          case "stashPending":
-            if (debug) console.log("---- STASH PENDING", action.type);
-            return skip(stashPending(currentState), action, nextFilterState);
-
-          case "cancelPending":
-            if (debug) console.log("---- CANCEL PENDING", action.type);
-            return skip(cancelPending(currentState), action, nextFilterState);
-
-          case "applyPending":
-            if (debug) console.log("---- APPLY PENDING", action.type);
-            return skip(applyPending(currentState), action, nextFilterState);
-
-          case "skip":
-          default:
-            return skip(currentState, action, nextFilterState);
-        }
-      }
-    }
-  };
-};
-
-function push(arr, val, limit = undefined) {
-  /*
-  functional array push, with a max length limit to the new array.
-  Like Array.push, except it returns new array and discards as needed
-  to enforce the length limit.
-  */
-  const narr = arr.slice(limit);
-  narr.push(val);
-  return narr;
-}
-
-export default Undoable;

client/src/reducers/undoableConfig.js

@@ -1,251 +0,0 @@
-import StateMachine from "../util/statemachine";
-import createFsmTransitions from "./undoableFsm";
-
-const actionKey = "@@undoable/filterAction";
-const stateKey = "@@undoable/filterState";
-
-/*
-these actions will not affect history
-*/
-const skipOnActions = new Set([
-  "url changed",
-  "interface reset started",
-  "initial data load start",
-  "configuration load complete",
-  "increment graph render counter",
-  "window resize",
-  "user reset start",
-  "reset colorscale",
-
-  "graph brush change",
-  "continuous metadata histogram brush",
-
-  "expression load start",
-  "expression load success",
-  "expression load error",
-
-  "request user defined gene started",
-  "request user defined gene success",
-  "clear all user defined genes",
-
-  "get single gene expression for coloring started",
-  "get single gene expression for coloring error"
-]);
-
-/*
-identical, repeated occurances of these action types will be debounced.
-Entire action must be identical (all keys).
-*/
-const debounceOnActions = new Set([
-  "color by categorical metadata",
-  "color by continuous metadata",
-  "color by expression"
-]);
-
-/*
-history will be cleared when these actions occur
-*/
-const clearOnActions = new Set([
-  "initial data load complete (universe exists)",
-  "reset World to eq Universe",
-  "initial data load error",
-  "user reset end"
-]);
-
-/*
-An immediate history save will be done for these
-*/
-const saveOnActions = new Set([
-  "categorical metadata filter select",
-  "categorical metadata filter deselect",
-  "categorical metadata filter all of these",
-  "categorical metadata none of these",
-
-  "color by categorical metadata",
-  "color by continuous metadata",
-  "color by expression",
-
-  "set scatterplot x",
-  "set scatterplot y",
-
-  "store current cell selection as differential set 1",
-  "store current cell selection as differential set 2",
-
-  "set World to current selection",
-  "set clip quantiles"
-]);
-
-/**
-StateMachine - processing complex action handling - see FSM graph for
-actual structure, in undoableFsm.js
-**/
-
-/*
-Default FSM actions.  Used to side-effect transitions in the graph.
-See graph definition for the transitions that use each.
-
-Signature:  (fsm, transition, reducerState, reducerAction) => undoableAction
-*/
-const stashPending = fsm => ({
-  [actionKey]: "stashPending",
-  [stateKey]: { fsm }
-});
-const cancelPending = () => ({
-  [actionKey]: "cancelPending",
-  [stateKey]: { fsm: null }
-});
-const applyPending = () => ({
-  [actionKey]: "applyPending",
-  [stateKey]: { fsm: null }
-});
-const skip = fsm => ({ [actionKey]: "skip", [stateKey]: { fsm } });
-const clear = () => ({ [actionKey]: "clear", [stateKey]: { fsm: null } });
-const save = fsm => ({ [actionKey]: "save", [stateKey]: { fsm } });
-
-/*
-Error handler for state transitions that are unexpected.  Called by
-StateMachine when it doesn't know what to do.
-
-Signature:  (fsm, event, from) => undoableAction
-*/
-const onFsmError = (fsm, event, from) => {
-  console.error(`FSM error [event: "${event}", state: "${from}"]`, fsm);
-  // In production, try to recover gracefully if we have unexpected state
-  return clear(fsm);
-};
-
-/*
-Definition of the transition graph mapping action types to history side effects.
-*/
-const fsmTransitions = createFsmTransitions(
-  stashPending,
-  cancelPending,
-  applyPending,
-  skip,
-  clear,
-  save
-);
-/* State machine we clone whenever we need to run it */
-const seedFsm = new StateMachine("init", fsmTransitions, onFsmError);
-
-/*
-See undoable.js for description action filter interface description.
-
-Basic approach:
-  * trivial handlers for skip, clear & save cases to keep config simple.
-  * only implement complex state machines where absolutely required (eg,
-    multi-event seleciton and the like)
-*/
-const actionFilter = debug => (state, action, prevFilterState) => {
-  const actionType = action.type;
-  const filterState = {
-    ...prevFilterState,
-    prevAction: action
-  };
-  if (skipOnActions.has(actionType)) {
-    return { [actionKey]: "skip", [stateKey]: filterState };
-  }
-  if (
-    debounceOnActions.has(actionType) &&
-    shallowObjectEq(action, prevFilterState.prevAction)
-  ) {
-    return { [actionKey]: "skip", [stateKey]: filterState };
-  }
-  if (clearOnActions.has(actionType)) {
-    return { [actionKey]: "clear", [stateKey]: filterState };
-  }
-  if (saveOnActions.has(actionType)) {
-    return { [actionKey]: "save", [stateKey]: filterState };
-  }
-
-  /*
-    Else, something more complex OR unknown to us....
-    */
-  if (seedFsm.events.has(actionType)) {
-    let { fsm } = filterState;
-    if (!fsm) {
-      /* no active FSM, so create one in init state */
-      fsm = seedFsm.clone("init");
-    }
-    return fsm.next(action.type, { state, action });
-  }
-
-  /* else, we have no idea what this is - skip it */
-  if (debug) console.log("**** ACTION FILTER EVENT HANDLER MISS", actionType);
-  return { [actionKey]: "skip", [stateKey]: filterState };
-};
-
-/*
-return true if objA and objB are ===, OR if:
-  - are both objects and not null
-  - have same own properties
-  - all values are strict equal (===)
-*/
-function shallowObjectEq(objA, objB) {
-  if (objA === objB) return true;
-  if (!objA || !objB) return false;
-  if (!shallowArrayEq(Object.keys(objA), Object.keys(objB))) return false;
-  if (!shallowArrayEq(Object.values(objA), Object.values(objB))) return false;
-  return true;
-}
-
-/*
-return true if arrA and arrB contain the same strict-equal values,
-in the same order.
-*/
-function shallowArrayEq(arrA, arrB) {
-  if (arrA.length !== arrB.length) return false;
-  for (let i = 0, l = arrA.length; i < l; i += 1) {
-    if (arrA[i] !== arrB[i]) return false;
-  }
-  return true;
-}
-
-/* configuration for the undoable meta reducer */
-const debug = false; // set truish for undoble debugging
-const undoableConfig = {
-  debug,
-  historyLimit: 50, // maximum history size
-  actionFilter: actionFilter(debug)
-};
-
-/*
-this code is strictly for sanity checking configuration, and is only
-enabled when we are debugging the undoable configuration (ie, debug === true).
-*/
-if (debug) {
-  /*
-  Confirm no intersection between the various trivial rejection action filters
-  */
-  if (
-    new Set([...skipOnActions].filter(x => clearOnActions.has(x))).size > 0 ||
-    new Set([...skipOnActions].filter(x => saveOnActions.has(x))).size > 0 ||
-    new Set([...clearOnActions].filter(x => saveOnActions.has(x))).size > 0
-  ) {
-    console.error(
-      "Undoable misconfiguration - action filters have redundant events"
-    );
-  }
-
-  /*
-  Confirm that no FSM events are blocked by a trivial rejection filter.
-  If this occurs, the FSM can't ever see the events needed to process
-  state transitions.
-  */
-  const trivialFilters = new Set([
-    ...skipOnActions,
-    ...clearOnActions,
-    ...saveOnActions
-  ]);
-  const trivialOverlapWithFsm = new Set(
-    [...trivialFilters].filter(x => seedFsm.events.has(x))
-  );
-  if (trivialOverlapWithFsm.size > 0) {
-    console.error(
-      "Undoable misconfiguration - trivival action filter blocking FSM filter",
-      [...trivialOverlapWithFsm]
-    );
-  }
-}
-
-export default undoableConfig;

client/src/reducers/undoableFsm.js

@@ -1,206 +0,0 @@
-/*
-State transition graph for complex action/history interactions.
-
-Assumed configuration from undoableConfig:
-    * By convention, "init" is used as the start state for all, and "done"
-      as the final state.
-    * Unexpected states will result in an error, plus a clear and cancelPending
-      side-effect.
-
-TODO: is is possible there is a more concise format for this, as it is
-a fairly repetitive pattern.
-
-These events are largely one of two types:
-a) async operations or multi-event options that should only be committed
-upon some success criteria, otherwise cancelled.
-
-b) compound actions that should be collapsed into a single history change.
-
-*/
-
-const createFsmTransitions = (
-  stashPending,
-  cancelPending,
-  applyPending,
-  skip,
-  clear,
-  save
-) => {
-  return [
-    /* graph selection brushing */
-    {
-      event: "graph brush start",
-      from: "init",
-      to: "graph brush in progress",
-      action: stashPending
-    },
-    {
-      event: "graph brush cancel",
-      from: "graph brush in progress",
-      to: "done",
-      action: applyPending
-    },
-    {
-      event: "graph brush deselect",
-      from: "graph brush in progress",
-      to: "done",
-      /* if current selection is all, cancelPending.  Else, applyPending */
-      action: (fsm, transition, data) =>
-        data.state.graphSelection.selection.mode === "all"
-          ? cancelPending()
-          : applyPending()
-    },
-    {
-      event: "graph brush end",
-      from: "graph brush in progress",
-      to: "done",
-      action: applyPending
-    },
-
-    /* graph selection lasso */
-    {
-      event: "graph lasso start",
-      from: "init",
-      to: "graph lasso in progress",
-      action: stashPending
-    },
-    {
-      event: "graph lasso cancel",
-      from: "graph lasso in progress",
-      to: "done",
-      action: applyPending
-    },
-    {
-      event: "graph lasso deselect",
-      from: "graph lasso in progress",
-      to: "done",
-      /* if current selection is all, cancelPending.  Else, applyPending */
-      action: (fsm, transition, data) =>
-        data.state.graphSelection.selection.mode === "all"
-          ? cancelPending()
-          : applyPending()
-    },
-    {
-      event: "graph lasso end",
-      from: "graph lasso in progress",
-      to: "done",
-      action: applyPending
-    },
-
-    /* Continuous metadata histogram brush selection */
-    {
-      event: "continuous metadata histogram start",
-      from: "init",
-      to: "continuous histo select in progress",
-      action: stashPending
-    },
-    {
-      event: "continuous metadata histogram cancel",
-      from: "continuous histo select in progress",
-      to: "done",
-      action: cancelPending
-    },
-    {
-      event: "continuous metadata histogram end",
-      from: "continuous histo select in progress",
-      to: "done",
-      action: applyPending
-    },
-
-    /* Single gene request by user */
-    {
-      event: "single user defined gene start",
-      from: "init",
-      to: "single user gene request in progress",
-      action: stashPending
-    },
-    {
-      event: "request user defined gene error",
-      from: "single user gene request in progress",
-      to: "single user gene error in progress",
-      action: skip
-    },
-    {
-      event: "single user defined gene error",
-      from: "single user gene error in progress",
-      to: "done",
-      action: cancelPending
-    },
-    {
-      event: "single user defined gene complete",
-      from: "single user gene request in progress",
-      to: "done",
-      action: applyPending
-    },
-
-    /* Bulk gene request by user */
-    {
-      event: "bulk user defined gene start",
-      from: "init",
-      to: "bulk user gene request in progress",
-      action: stashPending
-    },
-    {
-      event: "request user defined gene error",
-      from: "bulk user gene request in progress",
-      to: "bulk user gene request error in progress",
-      action: skip
-    },
-    {
-      event: "bulk user defined gene error",
-      from: "bulk user gene request error in progress",
-      to: "done",
-      action: cancelPending
-    },
-    {
-      event: "bulk user defined gene complete",
-      from: "bulk user gene request in progress",
-      to: "done",
-      action: applyPending
-    },
-
-    /* Compute Differential Expression button user action */
-    {
-      event: "request differential expression started",
-      from: "init",
-      to: "diffexp in progress",
-      action: stashPending
-    },
-    {
-      event: "request user defined gene error",
-      from: "diffexp in progress",
-      to: "done",
-      action: cancelPending
-    },
-    {
-      event: "request differential expression success",
-      from: "diffexp in progress",
-      to: "done",
-      action: applyPending
-    },
-
-    /* Clear Differential Expression button user action */
-    {
-      event: "clear differential expression",
-      from: "init",
-      to: "CDE Button in progress",
-      action: stashPending
-    },
-    {
-      event: "clear scatterplot",
-      from: "CDE Button in progress",
-      to: "done",
-      action: applyPending
-    },
-
-    /* clear scatter plot button (eg, on scatterplot view) */
-    {
-      event: "clear scatterplot",
-      from: "init",
-      to: "done",
-      action: save
-    }
-  ];
-};
-
-export default createFsmTransitions;

client/src/reducers/universe.js

@@ -1,47 +0,0 @@
-import { ControlsHelpers } from "../util/stateManager";
-
-const Universe = (state = null, action, nextSharedState, prevSharedState) => {
-  switch (action.type) {
-    case "initial data load complete (universe exists)": {
-      const { universe } = action;
-      return universe;
-    }
-
-    case "expression load success": {
-      let { varData } = state;
-
-      // Lazy load new expression data into the varData dataframe, if
-      // not already present.
-      //
-      Object.entries(action.expressionData).forEach(([key, val]) => {
-        // If not already in universe.varData, save entire expression column
-        if (!varData.hasCol(key)) {
-          varData = varData.withCol(key, val);
-        }
-      });
-
-      // Prune size of varData "cache" if getting out of hand....
-      //
-      const { userDefinedGenes, diffexpGenes } = prevSharedState;
-      const allTheGenesWeNeed = [
-        ...new Set(
-          userDefinedGenes,
-          diffexpGenes,
-          Object.keys(action.expressionData)
-        )
-      ];
-      varData = ControlsHelpers.pruneVarDataCache(varData, allTheGenesWeNeed);
-
-      return {
-        ...state,
-        varData
-      };
-    }
-
-    default: {
-      return state;
-    }
-  }
-};
-
-export default Universe;

client/src/reducers/world.js

@@ -1,131 +0,0 @@
-import { World, ControlsHelpers } from "../util/stateManager";
-import clip from "../util/clip";
-import quantile from "../util/quantile";
-
-const WorldReducer = (
-  state = null,
-  action,
-  nextSharedState,
-  prevSharedState
-) => {
-  switch (action.type) {
-    case "initial data load complete (universe exists)": {
-      const { universe } = nextSharedState;
-      const world = World.createWorldFromEntireUniverse(universe);
-      return world;
-    }
-
-    case "reset World to eq Universe": {
-      return prevSharedState.resetCache.world;
-    }
-
-    case "set World to current selection": {
-      /* Set viewable world to be the currently selected data */
-      const world = World.createWorldBySelection(
-        action.universe,
-        action.world,
-        action.crossfilter
-      );
-      return world;
-    }
-
-    case "set clip quantiles": {
-      const world = World.createWorldWithNewClip(
-        prevSharedState.universe,
-        state,
-        prevSharedState.crossfilter,
-        action.clipQuantiles
-      );
-      return world;
-    }
-
-    case "expression load success": {
-      const { universe } = nextSharedState;
-      const universeVarData = universe.varData;
-      let unclippedVarData = state.unclipped.varData;
-
-      // Lazy load new expression data into the unclipped varData dataframe, if
-      // not already present.
-      //
-      Object.entries(action.expressionData).forEach(([key, val]) => {
-        // If not already in world.varData, save sliced expression column
-        if (!unclippedVarData.hasCol(key)) {
-          // Slice if world !== universe, else just use whole column.
-          // Use the obsAnnotation index as the cut key, as we keep
-          // all world dataframes in sync.
-          let worldValSlice = val;
-          if (!World.worldEqUniverse(state, universe)) {
-            worldValSlice = universeVarData
-              .subset(state.obsAnnotations.rowIndex.keys(), [key], null)
-              .icol(0)
-              .asArray();
-          }
-
-          // Now build world's varData dataframe
-          unclippedVarData = unclippedVarData.withCol(
-            key,
-            worldValSlice,
-            state.obsAnnotations.rowIndex
-          );
-        }
-      });
-
-      // Prune size of varData unclipped dataframe if getting out of hand....
-      //
-      const { userDefinedGenes, diffexpGenes } = prevSharedState;
-      const allTheGenesWeNeed = [
-        ...new Set(
-          userDefinedGenes,
-          diffexpGenes,
-          Object.keys(action.expressionData)
-        )
-      ];
-      unclippedVarData = ControlsHelpers.pruneVarDataCache(
-        unclippedVarData,
-        allTheGenesWeNeed
-      );
-
-      // at this point, we have the unclipped data in unclippedVarData.
-      // Now create clipped.
-      //   - Drop columns no longer needed
-      //   - Add new columns
-      //
-      let clippedVarData = state.varData;
-      const keysToDrop = clippedVarData.colIndex
-        .keys()
-        .filter(k => !unclippedVarData.hasCol(k));
-      const keysToAdd = unclippedVarData.colIndex
-        .keys()
-        .filter(k => !clippedVarData.hasCol(k));
-      keysToDrop.forEach(k => {
-        clippedVarData = clippedVarData.dropCol(k);
-      });
-      keysToAdd.forEach(k => {
-        const data = unclippedVarData.col(k).asArray();
-        const q = [state.clipQuantiles.min, state.clipQuantiles.max];
-        const [qMinVal, qMaxVal] = quantile(q, data);
-        const clippedData = clip(data, qMinVal, qMaxVal, Number.NaN);
-        clippedVarData = clippedVarData.withCol(
-          k,
-          clippedData,
-          state.obsAnnotations.rowIndex
-        );
-      });
-
-      return {
-        ...state,
-        varData: clippedVarData,
-        unclipped: {
-          ...state.unclipped,
-          varData: unclippedVarData
-        }
-      };
-    }
-
-    default: {
-      return state;
-    }
-  }
-};
-
-export default WorldReducer;

client/src/util/actionHelpers.js

@@ -24,44 +24,23 @@ export function catchErrorsWrap(fn, dispatchToUser = false) {
   };
 }
 
-/*
-Wrapper to perform async fetch with some modest error handling
-and decoding.
-*/
-const doFetch = async (url, acceptType) => {
-  const res = await fetch(url, {
-    method: "get",
-    headers: new Headers({
-      Accept: acceptType
-    })
-  });
-  if (res.ok && res.headers.get("Content-Type").includes(acceptType)) {
-    return res;
-  }
-  // else an error
-  let msg = `Unexpected HTTP response ${res.status}, ${res.statusText}`;
-  const body = await res.text();
-  if (body && body.length > 0) {
-    msg = `${msg} -- ${body}`;
-  }
-  dispatchNetworkErrorMessageToUser(msg);
-  throw new Error(msg);
-};
-
 /*
 Wrapper to perform an async fetch and JSON decode response.
 */
 export const doJsonRequest = async url => {
-  const res = await doFetch(url, "application/json");
+  const res = await fetch(url, {
-  return res.json();
+    method: "get",
-};
+    headers: new Headers({
-
+      "Content-Type": "application/json"
-/*
+    })
-Wrapper to perform an async fetch for binary data.
+  });
-*/
+  if (res.ok && res.headers.get("Content-Type") === "application/json") {
-export const doBinaryRequest = async url => {
+    return res.json();
-  const res = await doFetch(url, "application/octet-stream");
+  }
-  return res.arrayBuffer();
+  // else an error
+  const msg = `Unexpected HTTP response ${res.status}, ${res.statusText}`;
+  dispatchNetworkErrorMessageToUser(msg);
+  throw new Error(msg);
 };
 
 /*

client/src/util/clip.js

@@ -1,25 +0,0 @@
-/*
-clip - clip all values in a Array or TypedArray, IN PLACE.
-
-Values in array are clipped if less than `lower` or greater than `upper`.
-
-If `setTo` is undefined, values less than `lower` will be set to `lower`,
-and values greater than `upper` will be set to `upper`.  
-
-If `setTo` is not undefined, values outside the [lower, upper] range will be set to 
-`setTo`.
-
-*/
-export default function clip(arr, lower, upper, setTo) {
-	const lowerSet = setTo === undefined ? lower : setTo;
-	const upperSet = setTo === undefined ? upper : setTo;
-	for (let i = 0, l = arr.length; i < l; i += 1) {
-		const v = arr[i];
-		if (v < lower) {
-			arr[i] = lowerSet;
-		} else if (v > upper) {
-			arr[i] = upperSet;
-		}
-	}
-	return arr;
-}

client/src/util/dataframe/dataframe.js

@@ -1,629 +0,0 @@
-import { IdentityInt32Index, isLabelIndex } from "./labelIndex";
-// weird cross-dependency that we should clean up someday...
-import { sortArray } from "../typedCrossfilter/sort";
-import { isTypedArray, isArrayOrTypedArray, callOnceLazy } from "./util";
-import { summarizeContinuous, summarizeCategorical } from "./summarize";
-
-/*
-Dataframe is an immutable 2D matrix similiar to Python Pandas Dataframe,
-but (currently) without all of the surrounding support functions.
-Data is stored in column-major layout, and each column is monomorphic.
-
-It supports:
-* Relatively efficient creation, cloning and subsetting
-* Very efficient columnar access (eg, sum down a column), and access
-  to the underlying column arrays.
-* Data access by row/col offset or label.  Labels are reasonably well
-  optimized for both numeric lables and arbitrary (eg, sting) labels.
-
-It does not currently support:
-* Views on matrix subset - for currently known access patterns,
-  it is more effiicent to copy on subsetting, optimizing for access
-  speed over memory use.
-* JS iterators - they are too slow.  Use explicit iteration over
-  offest or labels.
-
-Important assumptions embedded in the API:
-* Columns are implicitly categorical if they are a JS Array and numeric
-  (aka continuous) if they are a TypedArray.
-
-There are three index types for row/col indexing:
-* IdentityInt32Index - noop index, where the index label is the offset.
-* KeyIndex - index arbitrary JS objects.
-* DenseInt32Index - integer indexing.  Optimization over KeyIndex as it uses
-  Int32Array as a back-map to offsets.  This means that the index array
-  must be sized to [minLabel, maxLabel), so this is only useful when the label
-  range is relatively close the underlying offset range [minOffset, maxOffset).
-
-All private functions/methods/fields are prefixed by '__', eg, __compile().
-Don't use them outside of this file.
-
-Simple example:
-
-  // default indexing is integer offset.
-  const df = Dataframe.create([2,2], [['a', 'b'], [0, 1]])
-  console.log(df.at(0,0));  // outputs: a
-  console.log(df.col(1).asArray());   // outputs: [0, 1]
-
-  // KeyIndex
-  const df = new Dataframe([1,2], [['a'], ['b']], null, new KeyIndex(['A', 'B']))
-  console.log(df.at(0, 'A')); // outputs: a
-  console.log(df.col('A').asArray(); // outputs: ['a']
-
-Performance tuning is primarily focused on columnar access patterns, which is the
-dominant pattern in cellxgene.
-*/
-
-/**
-Dataframe
-**/
-
-class Dataframe {
-  /**
-  Constructors & factories
-  **/
-
-  constructor(
-    dims,
-    columnarData,
-    rowIndex = null,
-    colIndex = null,
-    __columnsAccessor = [] // private interface
-  ) {
-    /*
-    The base constructor is relatively hard to use - as an alternative,
-    see factory methods and clone/slice, below.
-
-    Parameters:
-      * dims - 2D array describing intendend dimensionality: [nRows,nCols].
-      * columnarData - JS array, nCols in length, containing array
-        or TypedArray of length nRows.
-      * rowIndex/colIndex - null (create default index using offsets as key),
-        or a caller-provided index.
-      * __columnsAccessor - private interface, do not specify.  Used internally
-        to improve caching of column accessors when possible (eg, clone(), 
-        dropCol(), withCol()).
-    All columns and indices must have appropriate dimensionality.
-    */
-    const [nRows, nCols] = dims;
-    if (nRows < 0 || nCols < 0) {
-      throw new RangeError("Dataframe dimensions must be positive");
-    }
-    if (!rowIndex) {
-      rowIndex = new IdentityInt32Index(nRows);
-    }
-    if (!colIndex) {
-      colIndex = new IdentityInt32Index(nCols);
-    }
-    Dataframe.__errorChecks(dims, columnarData, rowIndex, colIndex);
-
-    this.__columns = Array.from(columnarData);
-    this.dims = dims;
-    this.length = nRows; // convenience accessor for row dimension
-    this.rowIndex = rowIndex;
-    this.colIndex = colIndex;
-
-    this.__compile(__columnsAccessor);
-  }
-
-  static __errorChecks(dims, columnarData, rowIndex, colIndex) {
-    const [nRows, nCols] = dims;
-
-    /* check for expected types */
-    if (!Array.isArray(columnarData)) {
-      throw new TypeError("Dataframe constructor requires array of columns");
-    }
-    if (!columnarData.every(c => isArrayOrTypedArray(c))) {
-      throw new TypeError("Dataframe columns must all be Array or TypedArray");
-    }
-    if (!isLabelIndex(rowIndex)) {
-      throw new TypeError("Dataframe rowIndex is an unsupported type.");
-    }
-    if (!isLabelIndex(colIndex)) {
-      throw new TypeError("Dataframe colIndex is an unsupported type.");
-    }
-
-    /* check for expected dimensionality / size */
-    if (
-      nCols !== columnarData.length ||
-      !columnarData.every(c => c.length === nRows)
-    ) {
-      throw new RangeError(
-        "Dataframe dimension does not match provided data shape"
-      );
-    }
-    if (nRows !== rowIndex.size()) {
-      throw new RangeError(
-        "Dataframe rowIndex must have same size as underlying data"
-      );
-    }
-    if (nCols !== colIndex.size()) {
-      throw new RangeError(
-        "Dataframe colIndex must have same size as underlying data"
-      );
-    }
-  }
-
-  static __compileColumn(column, getOffset, getLabel) {
-    /*
-      Each column accessor is a function which will lookup data by
-      index (ie, is equivalent to dataframe.get(row, col), where 'col'
-      is fixed.
-
-      In addition, each column accessor has several functions:
-
-      asArray() -- return the entire column as a native Array or TypedArray.
-        Crucially, this native array only supports label indexing.
-        Example:
-          const arr = df.col('a').asArray();
-
-      has(rlabel) -- return boolean indicating of the row label
-        is contained within the column.  Example:
-          const isInColumn = df.col('a').includes(99)
-        For the default offset indexing, this is identical to:
-          const isInColumn = (99 > 0) && (99 < df.nRows);
-
-      ihas(roffset) -- same as has(), but accepts a row offset
-        instead of a row label.
-
-      indexOf(value) -- return the label (not offset) of the first instance of
-        'value' in the column.  If you want the offset, just use the builtin JS
-        indexOf() function, available on both Array and TypedArray.
-
-      iget(offset) -- return the value at 'offset'
-
-    */
-    const { length } = column;
-
-    /* get value by row label */
-    const get = function get(rlabel) {
-      return column[getOffset(rlabel)];
-    };
-
-    /* get value by row offset */
-    const iget = function iget(roffset) {
-      return column[roffset];
-    };
-
-    /* full column array access */
-    const asArray = function asArray() {
-      return column;
-    };
-
-    /* test for row label inclusion in column */
-    const has = function has(rlabel) {
-      const offset = getOffset(rlabel);
-      return offset >= 0 && offset < length;
-    };
-
-    const ihas = function ihas(offset) {
-      return offset >= 0 && offset < length;
-    };
-
-    /*
-    return first label (index) at which the value is found in this column,
-    or undefined if not found.
-
-    NOTE: not found return is DIFFERENT than the default Array.indexOf as
-    -1 is a plausible Dataframe row/col label.
-    */
-    const indexOf = function indexOf(value) {
-      const offset = column.indexOf(value);
-      if (offset === -1) {
-        return undefined;
-      }
-      return getLabel(offset);
-    };
-
-    /*
-    Summarize the column data. Lazy eval;
-    */
-    const summarize = callOnceLazy(() =>
-      isTypedArray(column)
-        ? summarizeContinuous(column)
-        : summarizeCategorical(column)
-    );
-
-    get.summarize = summarize;
-    get.asArray = asArray;
-    get.has = has;
-    get.ihas = ihas;
-    get.indexOf = indexOf;
-    get.iget = iget;
-    return get;
-  }
-
-  __compile(accessors) {
-    /*
-    Compile data accessors for each column.
-
-    Use an existing accessor if provided, else compile a new one.
-    */
-    const { getOffset, getLabel } = this.rowIndex;
-    this.__columnsAccessor = this.__columns.map((column, idx) => {
-      if (accessors[idx]) {
-        return accessors[idx];
-      }
-      return Dataframe.__compileColumn(column, getOffset, getLabel);
-    });
-  }
-
-  clone() {
-    /*
-    Clone this dataframe
-    */
-    return new this.constructor(
-      this.dims,
-      [...this.__columns],
-      this.rowIndex,
-      this.colIndex,
-      [...this.__columnsAccessor]
-    );
-  }
-
-  withCol(label, colData, withRowIndex = null) {
-    /*
-    Create a new DF, which is `this` plus the new column. Example:
-    const newDf = df.withCol("foo", [1,2,3]);
-
-    Dimensionality of new column must match existing dataframe.
-
-    Special case: empty dataframe will accept any size column.  Example:
-    const newDf = Dataframe.empty().withCol("foo", [1,2,3]);
-
-    If `withRowIndex` specified, the provided index will become the
-    rowIndex for the newly created dataframe.   If not specified,
-    the rowIndex from `this` will be used (ie, the rowIndex is
-    unchanged).
-    */
-    let dims;
-    let rowIndex;
-    if (this.isEmpty()) {
-      dims = [colData.length, 1];
-      rowIndex = null;
-    } else {
-      dims = [this.dims[0], this.dims[1] + 1];
-      ({ rowIndex } = this);
-    }
-
-    if (withRowIndex) {
-      rowIndex = withRowIndex;
-    }
-
-    const columns = [...this.__columns];
-    columns.push(colData);
-    const colIndex = this.colIndex.withLabel(label);
-    const columnsAccessor = [...this.__columnsAccessor];
-    return new this.constructor(
-      dims,
-      columns,
-      rowIndex,
-      colIndex,
-      columnsAccessor
-    );
-  }
-
-  dropCol(label) {
-    /*
-    Create a new dataframe, omitting one columns.
-
-    const newDf = df.dropCol("colors");
-    */
-    const dims = [this.dims[0], this.dims[1] - 1];
-    const coffset = this.colIndex.getOffset(label);
-    const columns = [...this.__columns];
-    columns.splice(coffset, 1);
-    const colIndex = this.colIndex.dropLabel(label);
-    const columnsAccessor = [...this.__columnsAccessor];
-    columnsAccessor.splice(coffset, 1);
-    return new this.constructor(
-      dims,
-      columns,
-      this.rowIndex,
-      colIndex,
-      columnsAccessor
-    );
-  }
-
-  static empty(rowIndex = null, colIndex = null) {
-    return new Dataframe([0, 0], [], rowIndex, colIndex);
-  }
-
-  static create(dims, columnarData) {
-    /*
-    Create a dataframe from raw columnar data.  All column arrays
-    must have the same length.   Identity indexing will be used.
-
-    Example:
-    const df = Dataframe.create([2,2], [new Uint32Array(2), new Float32Array(2)]);
-    */
-    return new Dataframe(dims, columnarData, null, null);
-  }
-
-  __subset(rowOffsets, colOffsets, withRowIndex) {
-    const dims = [...this.dims];
-
-    const getSortedLabelAndOffsets = (offsets, index) => {
-      /*
-      Given offsets, return both offsets and associated lables,
-      sorted by offset.
-      */
-      if (!offsets) {
-        return [null, null];
-      }
-      const sortedOffsets = sortArray(offsets);
-      const sortedLabels = new Array(sortedOffsets.length);
-      for (let i = 0, l = sortedOffsets.length; i < l; i += 1) {
-        sortedLabels[i] = index.getLabel(sortedOffsets[i]);
-      }
-      return [sortedLabels, sortedOffsets];
-    };
-
-    let { colIndex } = this;
-    if (colOffsets) {
-      let colLabels;
-      [colLabels, colOffsets] = getSortedLabelAndOffsets(
-        colOffsets,
-        this.colIndex
-      );
-      dims[1] = colOffsets.length;
-      colIndex = this.colIndex.subsetLabels(colLabels);
-    }
-
-    let { rowIndex } = this;
-    if (withRowIndex) rowIndex = withRowIndex;
-    if (rowOffsets) {
-      let rowLabels;
-      [rowLabels, rowOffsets] = getSortedLabelAndOffsets(
-        rowOffsets,
-        this.rowIndex
-      );
-      dims[0] = rowLabels.length;
-      if (!withRowIndex) rowIndex = this.rowIndex.subsetLabels(rowLabels);
-    }
-
-    /* subset columns */
-    let columns = this.__columns;
-    if (colOffsets) {
-      columns = new Array(colOffsets.length);
-      for (let i = 0, l = colOffsets.length; i < l; i += 1) {
-        columns[i] = this.__columns[colOffsets[i]];
-      }
-    }
-
-    /* subset rows */
-    if (rowOffsets) {
-      columns = columns.map(col => {
-        const newCol = new col.constructor(rowOffsets.length);
-        for (let i = 0, l = rowOffsets.length; i < l; i += 1) {
-          newCol[i] = col[rowOffsets[i]];
-        }
-        return newCol;
-      });
-    }
-    return new Dataframe(dims, columns, rowIndex, colIndex);
-  }
-
-  subset(rowLabels, colLabels = null, withRowIndex = null) {
-    /*
-    Subset by row/col labels.
-
-    withRowIndex allows assignment of new row index during subset operation.
-    If withRowIndex === null, it will reset the index to identity (offset)
-    indexing.  if withRowIndex is a label index object, it will be used
-    for the new dataframe.
-    */
-    const toOffsets = (labels, index) => {
-      if (!labels) {
-        return null;
-      }
-      return labels.map(label => {
-        const off = index.getOffset(label);
-        if (off === undefined) {
-          throw new RangeError(`unknown label: ${label}`);
-        }
-        return off;
-      });
-    };
-
-    const rowOffsets = toOffsets(rowLabels, this.rowIndex);
-    const colOffsets = toOffsets(colLabels, this.colIndex);
-    return this.__subset(rowOffsets, colOffsets, withRowIndex);
-  }
-
-  isubset(rowOffsets, colOffsets = null, withRowIndex = null) {
-    /*
-    Subset by row/col offset.
-
-    withRowIndex allows assignment of new row index during subset operation.
-    If withRowIndex === null, it will reset the index to identity (offset)
-    indexing.  if withRowIndex is a label index object, it will be used
-    for the new dataframe.
-    */
-    return this.__subset(rowOffsets, colOffsets, withRowIndex);
-  }
-
-  isubsetMask(rowMask, colMask = null, withRowIndex = null) {
-    /*
-    Subset on row/column based upon a truthy/falsey array (a mask).
-
-    withRowIndex allows assignment of new row index during subset operation.
-    If withRowIndex === null, it will reset the index to identity (offset)
-    indexing.  if withRowIndex is a label index object, it will be used
-    for the new dataframe.
-    */
-    const [nRows, nCols] = this.dims;
-    if (
-      (rowMask && rowMask.length !== nRows) ||
-      (colMask && colMask.length !== nCols)
-    ) {
-      throw new RangeError("boolean arrays must match row/col dimensions");
-    }
-
-    /* convert masks to lists - method wastes space, but is fast */
-    const toList = (mask, maxSize) => {
-      if (!mask) {
-        return null;
-      }
-      const list = new Int32Array(maxSize);
-      let elems = 0;
-      for (let i = 0, l = mask.length; i < l; i += 1) {
-        if (mask[i]) {
-          list[elems] = i;
-          elems += 1;
-        }
-      }
-      return new Int32Array(list.buffer, 0, elems);
-    };
-    const rowOffsets = toList(rowMask, nRows);
-    const colOffsets = toList(colMask, nCols);
-    return this.__subset(rowOffsets, colOffsets, withRowIndex);
-  }
-
-  /**
-  Data access with row/col.
-  **/
-
-  col(columnLabel) {
-    /*
-    Return accessor bound to a column.  Allows random row access
-    based upon the row indexing.  Returns undefined if the
-    columnLabel is not present in the dataframe.
-
-    Example for a dataframe with string labeled columns, and
-    default (offset) indices for rows (eg, [0, 'foo'])
-
-      const getValue = df.col('foo');
-      for (let r = 0; r < df.nRows; r += 1) {
-        console.log(r, getValue(r));
-      }
-
-    See __compile() for the functions available in a column accessor.
-    */
-    const coff = this.colIndex.getOffset(columnLabel);
-    return this.__columnsAccessor[coff];
-  }
-
-  icol(columnOffset) {
-    /*
-    Return column accessor by offset.
-    */
-    return this.__columnsAccessor[columnOffset];
-  }
-
-  at(r, c) {
-    /*
-    Access a single value, for a row/col label pair.
-
-    For performance reasons, there are no bounds or existance
-    checks on labels, and no defined behavior when these are supplied.
-    May return undefined, throw an Error, or do something else for
-    non-existant labels.  If you want predictable out-of-bounds
-    behavior, use has(), eg,
-
-    const myVal = df.has(r,l) ? df.at(r,l) : undefined;
-    */
-    const coff = this.colIndex.getOffset(c);
-    const roff = this.rowIndex.getOffset(r);
-    return this.__columns[coff][roff];
-  }
-
-  iat(r, c) {
-    /*
-    Access a single value, for a row/col offset (integer) position.
-
-    For performance reasons, there are no bounds checks on row/col offsets
-    or other well-defined behavior for out-of-bounds values.  If you want
-    well-defined bounds checking, use ihas(), eg,
-
-    const myVal = df.ihas(r, c) ? df.iat(r, c) : undefined;
-    */
-    return this.__columns[c][r];
-  }
-
-  has(r, c) {
-    /*
-    Test if row/col labels exist in the dataframe - returns true/false
-    */
-    const [nRows, nCols] = this.dims;
-    const coff = this.colIndex.getOffset(c);
-    const roff = this.rowIndex.getOffset(r);
-    return coff >= 0 && coff < nCols && roff >= 0 && roff < nRows;
-  }
-
-  ihas(r, c) {
-    /*
-    Test if row/col offset (integer) position exists in the
-    dataframe - returns true/false
-    */
-    const [nRows, nCols] = this.dims;
-    return c >= 0 && c < nCols && r >= 0 && r < nRows;
-  }
-
-  hasCol(c) {
-    /*
-    Test if col label exists - return true/false
-    */
-    return !!this.col(c);
-  }
-
-  isEmpty() {
-    /*
-    Return true if this is an empty dataframe, ie, has dimensions [0,0]
-    */
-    const [rows, cols] = this.dims;
-    return rows === 0 && cols === 0;
-  }
-
-  /****
-  Functional (map/reduce/etc) data access
-
-  TODO: most are not yet implemented, as there is no clear use case.   Can easily
-  add these as useful.
-  ****/
-
-  mapColumns(callback) {
-    /*
-    map all columns in the dataframe, returning a new dataframe comprised of the
-    return values, with the same index as the original dataframe.
-
-    callback MUST not modify the column, but instead return a mutated copy.
-    */
-    const columns = this.__columns.map(callback);
-    const columnsAccessor = columns.map((c, idx) =>
-      this.__columns[idx] === c ? this.__columnsAccessor[idx] : undefined
-    );
-    return new this.constructor(
-      this.dims,
-      columns,
-      this.rowIndex,
-      this.colIndex,
-      columnsAccessor
-    );
-  }
-
-  /*
-  Map & reduce of column or row
-
-  TODO remainder of map/reduce functions:  mapCol, mapRow, reduceRow, ...
-  */
-  /* comment out until we have a use for this
-
-  reduceCol(clabel, callback, initialValue) {
-    const coff = this.colIndex.getOffset(clabel);
-    const column = this.__columns[coff];
-    let start = 0;
-    let acc = initialValue;
-    if (initialValue === undefined) {
-      acc = column[0];
-      start = 1;
-    }
-    for (let i = start, l = column.length; i < l; i += 1) {
-      acc = callback(acc, column[i]);
-    }
-    return acc;
-  }
-  */
-}
-
-export default Dataframe;

client/src/util/dataframe/index.js

@@ -1,2 +0,0 @@
-export { default as Dataframe } from "./dataframe";
-export { DenseInt32Index, IdentityInt32Index, KeyIndex } from "./labelIndex";

client/src/util/dataframe/labelIndex.js

@@ -1,238 +0,0 @@
-/**
-Label indexing - map a label to & from an integer offset.  See Dataframe
-for how this is used.
-**/
-
-import { rangeFill as fillRange } from "../range";
-
-/*
-Private utility functions
-*/
-function extent(tarr) {
-  let min = 0x7fffffff;
-  let max = ~min; // eslint-disable-line no-bitwise
-  for (let i = 0, l = tarr.length; i < l; i += 1) {
-    const v = tarr[i];
-    if (v < min) {
-      min = v;
-    }
-    if (v > max) {
-      max = v;
-    }
-  }
-  return [min, max];
-}
-
-/* eslint-disable class-methods-use-this */
-class IdentityInt32Index {
-  /*
-  identity/noop index, with small assumptions that labels are int32
-  */
-  constructor(maxOffset) {
-    this.maxOffset = maxOffset;
-  }
-
-  keys() {
-    // memoize
-    const k = fillRange(new Int32Array(this.maxOffset));
-    this.keys = function keys() {
-      return k;
-    };
-    return k;
-  }
-
-  getOffset(i) {
-    // label to offset
-    return i;
-  }
-
-  getLabel(i) {
-    // offset to label
-    return i;
-  }
-
-  size() {
-    return this.maxOffset;
-  }
-
-  __promote(labelArray) {
-    /*
-    time/space decision - based on the resulting density
-    */
-    const [minLabel, maxLabel] = extent(labelArray);
-    const labelSpaceSize = maxLabel - minLabel + 1;
-    const density = labelSpaceSize / this.maxOffset;
-    /* 0.1 is a magic number, that needs testing to optimize */
-    if (density < 0.1) {
-      return new KeyIndex(labelArray);
-    }
-    return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
-  }
-
-  subsetLabels(labelArray) {
-    return this.__promote(labelArray);
-  }
-
-  withLabel(label) {
-    if (label === this.maxOffset) {
-      return new IdentityInt32Index(label + 1);
-    }
-    return this.__promote([...this.keys(), label]);
-  }
-
-  dropLabel(label) {
-    if (label === this.maxOffset - 1) {
-      return new IdentityInt32Index(label);
-    }
-    const labelArray = [...this.keys()];
-    labelArray.splice(labelArray.indexOf(label), 1);
-    return this.__promote(labelArray);
-  }
-}
-/* eslint-enable class-methods-use-this */
-
-/* eslint-disable class-methods-use-this */
-class DenseInt32Index {
-  /*
-  DenseInt32Index indexes integer labels, and uses Int32Array typed arrays
-  for both forward and reverse indexing.   This means that the min/max range
-  of the forward index labels must be known a priori (so that the index
-  array can be pre-allocated).
-  */
-  constructor(labels, labelRange = null) {
-    if (labels.constructor !== Int32Array) {
-      labels = new Int32Array(labels);
-    }
-
-    if (!labelRange) {
-      labelRange = extent(labels);
-    }
-    const [minLabel, maxLabel] = labelRange;
-    const labelSpaceSize = maxLabel - minLabel + 1;
-    const index = new Int32Array(labelSpaceSize).fill(-1);
-    for (let i = 0, l = labels.length; i < l; i += 1) {
-      const label = labels[i];
-      index[label - minLabel] = i;
-    }
-
-    this.minLabel = minLabel;
-    this.rindex = labels;
-    this.index = index;
-    this.__compile();
-  }
-
-  __compile() {
-    const { minLabel, index, rindex } = this;
-    this.getOffset = function getOffset(l) {
-      return index[l - minLabel];
-    };
-    this.getLabel = function getLabel(i) {
-      return rindex[i];
-    };
-  }
-
-  keys() {
-    return this.rindex;
-  }
-
-  size() {
-    return this.rindex.length;
-  }
-
-  __promote(labelArray) {
-    /*
-    time/space decision - if we are going to use less than 10% of the
-    dense index space, switch to a KeyIndex (which is slower, but uses
-    less memory for sparse label spaces).
-    */
-    const [minLabel, maxLabel] = extent(labelArray);
-    const labelSpaceSize = maxLabel - minLabel + 1;
-    const density = labelSpaceSize / this.rindex.length;
-    /* 0.1 is a magic number, that needs testing to optimize */
-    if (density < 0.1) {
-      return new KeyIndex(labelArray);
-    }
-    return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
-  }
-
-  subsetLabels(labelArray) {
-    return this.__promote(labelArray);
-  }
-
-  withLabel(label) {
-    return this.__promote([...this.keys(), label]);
-  }
-
-  dropLabel(label) {
-    const labelArray = [...this.keys()];
-    labelArray.splice(labelArray.indexOf(label), 1);
-    return this.__promote(labelArray);
-  }
-}
-/* eslint-enable class-methods-use-this */
-
-/* eslint-disable class-methods-use-this */
-class KeyIndex {
-  /*
-  KeyIndex indexes arbitrary JS primitive types, and uses a Map()
-  as its core data structure.
-  */
-  constructor(labels) {
-    const index = new Map();
-    if (labels === undefined) {
-      labels = [];
-    }
-    const rindex = labels;
-    labels.forEach((v, i) => {
-      index.set(v, i);
-    });
-
-    this.index = index;
-    this.rindex = rindex;
-    this.__compile();
-  }
-
-  __compile() {
-    const { index, rindex } = this;
-    this.getOffset = function getOffset(k) {
-      return index.get(k);
-    };
-    this.getLabel = function getLabel(i) {
-      return rindex[i];
-    };
-  }
-
-  keys() {
-    return this.rindex;
-  }
-
-  size() {
-    return this.rindex.length;
-  }
-
-  subsetLabels(labelArray) {
-    return new KeyIndex(labelArray);
-  }
-
-  withLabel(label) {
-    return new KeyIndex([...this.rindex, label]);
-  }
-
-  dropLabel(label) {
-    const idx = this.rindex.indexOf(label);
-    const labelArray = [...this.rindex];
-    labelArray.splice(idx, 1);
-    return new KeyIndex(labelArray);
-  }
-}
-/* eslint-enable class-methods-use-this */
-
-function isLabelIndex(i) {
-  return (
-    i instanceof IdentityInt32Index ||
-    i instanceof DenseInt32Index ||
-    i instanceof KeyIndex
-  );
-}
-
-export { DenseInt32Index, IdentityInt32Index, KeyIndex, isLabelIndex };

client/src/util/dataframe/summarize.js

@@ -1,82 +0,0 @@
-/*
-Private dataframe support functions
-
-TODO / XXX: for scalar/continuous data, this uses a naive method
-of computing quantiles.  Would be good to switch from sort to 
-partition at some point.
-*/
-
-import quantile from "../quantile";
-import { sortArray } from "../typedCrossfilter/sort";
-
-// [ 0, 0.01, 0.02, ..., 1.0]
-const centileNames = new Array(101).fill(0).map((v, idx) => idx / 100);
-
-export function summarizeContinuous(col) {
-  let min;
-  let max;
-  let nan = 0;
-  let pinf = 0;
-  let ninf = 0;
-  let percentiles;
-  if (col) {
-    // -Inf < finite < Inf < NaN
-    const sortedCol = sortArray(new col.constructor(col));
-
-    // count non-finites, which are at each end of sorted data
-    for (let i = sortedCol.length - 1; i >= 0; i -= 1) {
-      if (!Number.isNaN(sortedCol[i])) {
-        nan = sortedCol.length - i - 1;
-        break;
-      }
-    }
-    for (let i = 0, l = sortedCol.length; i < l; i += 1) {
-      if (sortedCol[i] !== Number.NEGATIVE_INFINITY) {
-        ninf = i;
-        break;
-      }
-    }
-    for (let i = sortedCol.length - nan - 1; i >= 0; i -= 1) {
-      if (sortedCol[i] !== Number.POSITIVE_INFINITY) {
-        pinf = sortedCol.length - i - nan - 1;
-        break;
-      }
-    }
-
-    // compute percentiles on finite data ONLY
-    const sortedColFiniteOnly = sortedCol.slice(
-      ninf,
-      sortedCol.length - nan - pinf
-    );
-    percentiles = quantile(centileNames, sortedColFiniteOnly, true);
-    min = percentiles[0];
-    max = percentiles[100];
-  }
-  return {
-    categorical: false,
-    min,
-    max,
-    nan,
-    pinf,
-    ninf,
-    percentiles
-  };
-}
-
-export function summarizeCategorical(col) {
-  const categoryCounts = new Map();
-  if (col) {
-    for (let r = 0, l = col.length; r < l; r += 1) {
-      const val = col[r];
-      let curCount = categoryCounts.get(val);
-      if (curCount === undefined) curCount = 0;
-      categoryCounts.set(val, curCount + 1);
-    }
-  }
-  return {
-    categorical: true,
-    categories: [...categoryCounts.keys()],
-    categoryCounts,
-    numCategories: categoryCounts.size
-  };
-}

client/src/util/dataframe/util.js

@@ -1,19 +0,0 @@
-/*
-Private utility code for dataframe
-*/
-
-export { isTypedArray, isArrayOrTypedArray } from "../typeHelpers";
-
-export function callOnceLazy(f) {
-  let value;
-  let calledOnce = false;
-  const result = function result(...args) {
-    if (!calledOnce) {
-      value = f(...args);
-      calledOnce = true;
-    }
-    return value;
-  };
-
-  return result;
-}

client/src/util/finiteExtent.js

@@ -1,33 +0,0 @@
-/*
-Return the [minimum, maximum] extent, of the given typed array, ignoring
-non-finite values (ie, +Infinity, -Infinity).
-
-If undefined or empty array, or array contains only non-finite numbers,
-will return [undefined, undefined]
-*/
-
-function finiteExtent(tarr) {
-  let min;
-  let max;
-  let i;
-
-  for (i = 0; i < tarr.length; i += 1) {
-    const val = tarr[i];
-    if (Number.isFinite(val)) {
-      min = val;
-      max = val;
-      i += 1;
-      break;
-    }
-  }
-  for (; i < tarr.length; i += 1) {
-    const val = tarr[i];
-    if (Number.isFinite(val)) {
-      if (min > val) min = val;
-      if (max < val) max = val;
-    }
-  }
-  return [min, max];
-}
-
-export default finiteExtent;

client/src/util/fromEntries.js

@@ -1,13 +0,0 @@
-export default function fromEntries(arr) {
-	/*
-	Similar to Object.fromEntries, but only handles array.
-	This could be replaced with the standard fucnction once it
-	is widely available.   As of 3/20/2019, it has not yet
-	been released in the Chrome stable channel.
-	*/
-	const obj = {};
-	for (let i = 0, l = arr.length; i < l; i += 1) {
-		obj[arr[i][0]] = arr[i][1];
-	}
-	return obj;
-}

client/src/util/quantile.js

@@ -1,29 +0,0 @@
-/*
-quantiles - calculate quantiles for the typed array.
-
-Currently interpolates to 'lower' value.
-
-Arguments:
-
-	* quantArr - array of quantiles to compute, where values:  0 <= value <= 1.0
-	* tarr - a typed array
-	* sorted - option bool.  If false (default), will assume array is not sorted.
-	 	If true, will assume it is sorted.
-
-*/
-
-import { sortArray } from "./typedCrossfilter/sort";
-
-export default function quantile(quantArr, tarr, sorted = false) {
-	/*
-	start with the naive (sort) implementation.  Later, use a faster partition
-	*/
-	const arr = sorted ? tarr : sortArray(new tarr.constructor(tarr)); // copy
-	const len = arr.length;
-	return quantArr.map(q => {
-		if (q === 1) {
-			return arr[len - 1];
-		}
-		return arr[Math.floor(q * len)];
-	});
-}

client/src/util/range.js

@@ -1,45 +0,0 @@
-/*
-Array range creation
-
-range(start, stop, step) -> Array
-	This is identical to https://docs.python.org/3/library/functions.html#func-range
-	Returns new array filled with a range of numbers.
-
-	Usage:
-
-		range(stop) - start defaults to zero, step defaults to 1
-		range(start, stop, [step]) - step defaults to 1
-
-	Examples:
-		range(3) -> [0, 1, 2]
-		range(1, 3) -> [1, 2]
-		range(1, 5, 2) -> [1, 3]
-
-
-rangeFill(array, start, step) -> array
-	Fill entire array with values, from start, by step.  Returns first array.
-	start defaults to zero, step defaults to one.
-
-*/
-
-function _doFill(arr, start, step, count) {
-	for (let idx = 0, val = start; idx < count; idx += 1, val += step) {
-		arr[idx] = val;
-	}
-	return arr;
-}
-
-export function rangeFill(arr, start = 0, step = 1) {
-	return _doFill(arr, start, step, arr.length);
-}
-
-export function range(start, stop, step) {
-	if (start === undefined) return [];
-	if (stop === undefined) {
-		stop = start;
-		start = 0;
-	}
-	step = step || 1; // catch undefind and zero
-	const len = Math.max(Math.ceil((stop - start) / step), 0);
-	return _doFill(new Array(len), start, step, len);
-}

client/src/util/stateManager/colorHelpers.js

@@ -1,120 +0,0 @@
-/*
-Helper functions for the embedded graph colors
-*/
-import * as d3 from "d3";
-import { interpolateRainbow, interpolateCool } from "d3-scale-chromatic";
-import * as globals from "../../globals";
-import parseRGB from "../parseRGB";
-import finiteExtent from "../finiteExtent";
-import { range } from "../range";
-
-/*
-create new colors state object.   Paramters:
-  - world - current world object
-  - mode - color-by mode.  One of: null, "color by expression",
-    "color by continuous metadata", "color by categorical metadata"
-  -
-*/
-function createColors(world, colorMode = null, colorAccessor = null) {
-  switch (colorMode) {
-    case "color by categorical metadata": {
-      return createColorsByCategoricalMetadata(world, colorAccessor);
-    }
-    case "color by continuous metadata": {
-      return createColorsByContinuousMetadata(world, colorAccessor);
-    }
-    case "color by expression": {
-      return createColorsByExpression(world, colorAccessor);
-    }
-    default: {
-      const defaultCellColor = parseRGB(globals.defaultCellColor);
-      return {
-        rgb: new Array(world.nObs).fill(defaultCellColor),
-        scale: undefined
-      };
-    }
-  }
-}
-
-function createColorsByCategoricalMetadata(world, accessor) {
-  const { categories } = world.schema.annotations.obsByName[accessor];
-
-  const scale = d3
-    .scaleSequential(interpolateRainbow)
-    .domain([0, categories.length]);
-
-  /* pre-create colors - much faster than doing it for each obs */
-  const colors = categories.reduce((acc, cat, idx) => {
-    acc[cat] = parseRGB(scale(idx));
-    return acc;
-  }, {});
-
-  const rgb = new Array(world.nObs);
-  const data = world.obsAnnotations.col(accessor).asArray();
-  for (let i = 0, len = world.obsAnnotations.length; i < len; i += 1) {
-    const cat = data[i];
-    rgb[i] = colors[cat];
-  }
-  return { rgb, scale };
-}
-
-function createColorsByContinuousMetadata(world, accessor) {
-  const colorBins = 100;
-  const col = world.obsAnnotations.col(accessor);
-  const { min, max } = col.summarize();
-  const scale = d3
-    .scaleQuantile()
-    .domain([min, max])
-    .range(range(colorBins - 1, -1, -1));
-
-  /* pre-create colors - much faster than doing it for each obs */
-  const colors = new Array(colorBins);
-  for (let i = 0; i < colorBins; i += 1) {
-    colors[i] = parseRGB(interpolateCool(i / colorBins));
-  }
-
-  const nonFiniteColor = parseRGB(globals.nonFiniteCellColor);
-  const rgb = new Array(world.nObs);
-  const data = col.asArray();
-  for (let i = 0, len = world.obsAnnotations.length; i < len; i += 1) {
-    const val = data[i];
-    if (Number.isFinite(val)) {
-      const c = scale(val);
-      rgb[i] = colors[c];
-    } else {
-      rgb[i] = nonFiniteColor;
-    }
-  }
-  return { rgb, scale };
-}
-
-function createColorsByExpression(world, accessor) {
-  const expression = world.varData.col(accessor).asArray();
-  const colorBins = 100;
-  const [min, max] = finiteExtent(expression);
-  const scale = d3
-    .scaleQuantile()
-    .domain([min, max])
-    .range(range(colorBins - 1, -1, -1));
-
-  /* pre-create colors - much faster than doing it for each obs */
-  const colors = new Array(colorBins);
-  for (let i = 0; i < colorBins; i += 1) {
-    colors[i] = parseRGB(interpolateCool(i / colorBins));
-  }
-  const nonFiniteColor = parseRGB(globals.nonFiniteCellColor);
-
-  const rgb = new Array(world.nObs);
-  for (let i = 0, len = expression.length; i < len; i += 1) {
-    const e = expression[i];
-    if (Number.isFinite(e)) {
-      const c = scale(e);
-      rgb[i] = colors[c];
-    } else {
-      rgb[i] = nonFiniteColor;
-    }
-  }
-  return { rgb, scale };
-}
-
-export default createColors;

client/src/util/stateManager/controlsHelpers.js

@@ -1,169 +0,0 @@
-/*
-Helper functions for the controls reducer
-*/
-
-import _ from "lodash";
-
-import * as globals from "../../globals";
-import { rangeFill as fillRange } from "../range";
-import {
-  userDefinedDimensionName,
-  diffexpDimensionName
-} from "../nameCreators";
-
-/*
-Selection state for categoricals are tracked in an Object that
-has two main components for each category:
-1. mapping of option value to an index
-2. array of bool selection state by index
-Remember that option values can be ANY js type, except undefined/null.
-
-  {
-    _category_name_1: {
-      // map of option value to index
-      categoryIndices: Map([
-        catval1: index,
-        ...
-      ])
-
-      // index->selection true/false state
-      categorySelected: [ true/false, true/false, ... ]
-
-      // number of options
-      numCategories: number,
-
-      // isTruncated - true if the options for selection has
-      // been truncated (ie, was too large to implement)
-    }
-  }
-*/
-function topNCategories(summary) {
-  const counts = _.map(summary.categories, cat =>
-    summary.categoryCounts.get(cat)
-  );
-  const sortIndex = fillRange(new Array(summary.numCategories)).sort(
-    (a, b) => counts[b] - counts[a]
-  );
-  const sortedCategories = _.map(sortIndex, i => summary.categories[i]);
-  const sortedCounts = _.map(sortIndex, i => counts[i]);
-  const N = globals.maxCategoricalOptionsToDisplay;
-
-  if (sortedCategories.length < N) {
-    return [sortedCategories, sortedCounts];
-  }
-  return [sortedCategories.slice(0, N), sortedCounts.slice(0, N)];
-}
-
-export function createCategoricalSelection(maxCategoryItems, world) {
-  const res = {};
-  _.forEach(world.obsAnnotations.colIndex.keys(), key => {
-    const summary = world.obsAnnotations.col(key).summarize();
-    if (summary.categories) {
-      const isColorField = key.includes("color") || key.includes("Color");
-      const isSelectableCategory =
-        !isColorField &&
-        key !== "name" &&
-        summary.categories.length < maxCategoryItems;
-      if (isSelectableCategory) {
-        const [categoryValues, categoryCounts] = topNCategories(summary);
-        const categoryIndices = new Map(categoryValues.map((v, i) => [v, i]));
-        const numCategories = categoryIndices.size;
-        const categorySelected = new Array(numCategories).fill(true);
-        const isTruncated = categoryValues.length < summary.numCategories;
-        res[key] = {
-          categoryValues, // array: of natively typed category values
-          categoryIndices, // map: category value (native type) -> category index
-          categorySelected, // array: t/f selection state
-          numCategories, // number: of categories
-          isTruncated, // bool: true if list was truncated
-          categoryCounts // array: cardinality of each category
-        };
-      }
-    }
-  });
-  return res;
-}
-
-/*
-given a categoricalSelection, return the list of all category values
-where selection state is true (ie, they are selected).
-*/
-export function selectedValuesForCategory(categorySelectionState) {
-  const selectedValues = _([...categorySelectionState.categoryIndices])
-    .filter(tuple => categorySelectionState.categorySelected[tuple[1]])
-    .map(tuple => tuple[0])
-    .value();
-  return selectedValues;
-}
-
-/*
-build a crossfilter dimensions for all gene expression related dimensions.
-*/
-export function createGeneDimensions(
-  userDefinedGenes,
-  diffexpGenes,
-  world,
-  crossfilter
-) {
-  crossfilter = userDefinedGenes.reduce(
-    (xflt, gene) =>
-      xflt.addDimension(
-        userDefinedDimensionName(gene),
-        "scalar",
-        world.varData.col(gene).asArray(),
-        Float32Array
-      ),
-    crossfilter
-  );
-  crossfilter = diffexpGenes.reduce(
-    (xflt, gene) =>
-      xflt.addDimension(
-        diffexpDimensionName(gene),
-        "scalar",
-        world.varData.col(gene).asArray(),
-        Float32Array
-      ),
-    crossfilter
-  );
-  return crossfilter;
-}
-
-export function pruneVarDataCache(varData, needed) {
-  /*
-  Remove any unneeded columns from the varData dataframe.  Will only
-  prune / remove if the total column count exceeds VarDataCacheLowWatermark
-
-  Note: this code leverages the fact that dataframe offsets indicate
-  the order in which the columns were added.  This crudely provides
-  LRU semantics, so we can delete "older" columns first.
-  */
-
-  /*
-  VarDataCacheLowWatermark - this cofig value sets the minimum cache size,
-  in columns, below which we don't throw away data.
-
-  The value should be high enough so we are caching the maximum which will
-  "typically" be used in the UI (currently: 10 for diffexp, and N for user-
-  specified genes), and low enough to account for memory use (any single
-  column size is 4 bytes * numObs, so a column can be multi-megabyte in common
-  use cases).
-  */
-  const VarDataCacheLowWatermark = 32;
-
-  const numOverWatermark = varData.dims[1] - VarDataCacheLowWatermark;
-  if (numOverWatermark <= 0) return varData;
-
-  const { colIndex } = varData;
-  const all = colIndex.keys();
-  const unused = _.difference(all, needed);
-  if (unused.length > 0) {
-    // sort by offset in the dataframe - ie, psuedo-LRU
-    unused.sort((a, b) => colIndex.getOffset(a) - colIndex.getOffset(b));
-    const numToDrop =
-      unused.length < numOverWatermark ? unused.length : numOverWatermark;
-    for (let i = 0; i < numToDrop; i += 1) {
-      varData = varData.dropCol(unused[i]);
-    }
-  }
-  return varData;
-}

client/src/util/stateManager/index.js

@@ -3,7 +3,7 @@
 /*
 Model manager providing an abstraction for the use of the reducer code.
 This module provides several buckets of functionality:
-  - schema and config driven tranformation of the wire protocol
+  - schema and config driven tranformation of the dataframe wire protocol
     into a format that is easy for the UI code to use.
   - manage the universe/world abstraction:
     + universe: all of the server-provided, read-only data
@@ -14,8 +14,6 @@ This is all VERY tightly integrated with reducers and actions, and
 exists to support those concepts.
 */
 
-export { default as createColors } from "./colorHelpers";
 export * as Universe from "./universe";
 export * as World from "./world";
-export * as WorldUtil from "./worldUtil";
+export * as kvCache from "./keyvalcache";
-export * as ControlsHelpers from "./controlsHelpers";

client/src/util/stateManager/keyvalcache.js

@@ -0,0 +1,122 @@
+// jshint esversion: 6
+import _ from "lodash";
+
+/*
+Very simple key/value cache for use by World & Universe.   Cache keys must
+be a string, and values are any JS non-primitive value.
+
+  * constructor(lowWatermark, minTTL):
+      - lowWatermark defines the number of cache elements below which
+        flushing will not occur.
+      - minTTL defines minimum time in milliseconds that cache entries will live.
+        A value of -1 disables automatic flushing (flush() can still
+        be called by external user).
+  * set() - add a key/val pair.
+  * get() - get a value or undefined if not present.
+  * flush(minAgeMs) - flush cache entries in excess of lowWatermark if those
+    entries are older than minAgeMs.
+
+*/
+
+const cachePrivateKey = "__kvcachekey__";
+const defaultLowWatermark = 32;
+const defaultMinTTL = 1000;
+
+function create(lowWatermark = defaultLowWatermark, minTTL = defaultMinTTL) {
+  if (typeof minTTL !== "number" || typeof lowWatermark !== "number") {
+    throw new TypeError(
+      "minTTL and lowWatermark parameters must be a primitive number"
+    );
+  }
+  if (lowWatermark < 0 || minTTL < 0) {
+    throw new RangeError(
+      "minTTL and lowWatermark parameters must be number greater than zero"
+    );
+  }
+
+  return {
+    [cachePrivateKey]: {
+      lowWatermark,
+      minTTL
+    }
+  };
+}
+
+function get(kvcache, key) {
+  if (key === cachePrivateKey) {
+    throw new RangeError(`key parameter may not have value ${cachePrivateKey}`);
+  }
+
+  const val = kvcache[key];
+  if (val) {
+    val[cachePrivateKey] = Date.now();
+  }
+  return val;
+}
+
+function set(kvcache, key, val) {
+  if (key === cachePrivateKey) {
+    throw new RangeError(`key parameter may not have value ${cachePrivateKey}`);
+  }
+
+  const newKvCache = { ...kvcache };
+  newKvCache[key] = val;
+  val[cachePrivateKey] = Date.now();
+  flushInPlace(newKvCache);
+  return newKvCache;
+}
+
+function flush(kvcache) {
+  const newKvCache = { ...kvcache };
+  flushInPlace(newKvCache);
+  return newKvCache;
+}
+
+/*
+Flush elements from cache IF cache size is greater than lowWatermark, and
+those elements are older than minAgeMS
+*/
+function flushInPlace(kvCache) {
+  const { lowWatermark, minTTL } = kvCache[cachePrivateKey];
+  const eol = Date.now() - minTTL;
+  const allKeys = _(kvCache)
+    .keys()
+    .filter(k => k !== cachePrivateKey)
+    .sortBy([k => kvCache[k][cachePrivateKey]])
+    .value();
+
+  if (allKeys.length > lowWatermark) {
+    const keysToDelete = _(allKeys)
+      .slice(0, allKeys.length - lowWatermark)
+      .filter(k => kvCache[k][cachePrivateKey] <= eol)
+      .value();
+    _.forEach(keysToDelete, k => delete kvCache[k]);
+  }
+
+  return kvCache;
+}
+
+/*
+use to create a cache that is a transformation of another cache.
+*/
+function map(srcKvCache, cb, createOptions) {
+  const keysInSrcKvCache = _(srcKvCache)
+    .keys()
+    .filter(k => k !== cachePrivateKey)
+    .value();
+  const lowWatermark = _.get(
+    createOptions,
+    "lowWatermark",
+    defaultLowWatermark
+  );
+  const minTTL = _.get(createOptions, "minTTL", defaultMinTTL);
+  const newKvCache = create(lowWatermark, minTTL);
+  _.forEach(keysInSrcKvCache, key => {
+    const val = cb(get(srcKvCache, key), key);
+    newKvCache[key] = val;
+    val[cachePrivateKey] = Date.now();
+  });
+  return newKvCache;
+}
+
+export { create, get, set, flush, map };

client/src/util/stateManager/matrix.js

@@ -1,75 +0,0 @@
-import { flatbuffers } from "flatbuffers";
-import { NetEncoding } from "./matrix_generated";
-
-const utf8Decoder = new TextDecoder("utf-8");
-
-/*
-Matrix flatbuffer decoding support.   See fbs/matrix.fbs
-*/
-
-/*
-Decode NetEncoding.TypedArray
-*/
-function decodeTypedArray(uType, uValF, inplace = false) {
-  if (uType === NetEncoding.TypedArray.NONE) {
-    return null;
-  }
-
-  // Convert to a JS class that supports this type
-  const TypeClass = NetEncoding[NetEncoding.TypedArray[uType]];
-  // Create a TypedArray that references the underlying buffer
-  let arr = uValF(new TypeClass()).dataArray();
-  if (uType === NetEncoding.TypedArray.JSONEncodedArray) {
-    const json = utf8Decoder.decode(arr);
-    arr = JSON.parse(json);
-  } else if (!inplace) {
-    /* force copy to release underlying FBS buffer */
-    arr = new arr.constructor(arr);
-  }
-  return arr;
-}
-
-/*
-Parameter: Uint8Array or ArrayBuffer containing raw flatbuffer Matrix
-Returns: object containing decoded Matrix:
-{
-  nRows: num,
-  nCols: num,
-  columns: [
-    each column, which will be a TypedArray or Array
-  ]
-  colIdx: []|null
-}
-*/
-function decodeMatrixFBS(arrayBuffer, inplace = false) {
-  const bb = new flatbuffers.ByteBuffer(new Uint8Array(arrayBuffer));
-  const df = NetEncoding.Matrix.getRootAsMatrix(bb);
-
-  const nRows = df.nRows();
-  const nCols = df.nCols();
-
-  /* decode columns */
-  const columnsLength = df.columnsLength();
-  const columns = Array(columnsLength).fill(null);
-  for (let c = 0; c < columnsLength; c += 1) {
-    const col = df.columns(c);
-    columns[c] = decodeTypedArray(col.uType(), col.u.bind(col), inplace);
-  }
-
-  /* decode col_idx */
-  const colIdx = decodeTypedArray(
-    df.colIndexType(),
-    df.colIndex.bind(df),
-    inplace
-  );
-
-  return {
-    nRows,
-    nCols,
-    columns,
-    colIdx,
-    rowIdx: null
-  };
-}
-
-export default decodeMatrixFBS;

client/src/util/stateManager/matrix_generated.js

@@ -1,835 +0,0 @@
-// automatically generated by the FlatBuffers compiler, do not modify
-
-/**
- * @const
- * @namespace
- */
-var NetEncoding = NetEncoding || {};
-
-/**
- * @enum
- */
-NetEncoding.TypedArray = {
-  NONE: 0, 0: 'NONE',
-  Float32Array: 1, 1: 'Float32Array',
-  Int32Array: 2, 2: 'Int32Array',
-  Uint32Array: 3, 3: 'Uint32Array',
-  Float64Array: 4, 4: 'Float64Array',
-  JSONEncodedArray: 5, 5: 'JSONEncodedArray'
-};
-
-/**
- * @constructor
- */
-NetEncoding.Float32Array = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {NetEncoding.Float32Array}
- */
-NetEncoding.Float32Array.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {NetEncoding.Float32Array=} obj
- * @returns {NetEncoding.Float32Array}
- */
-NetEncoding.Float32Array.getRootAsFloat32Array = function(bb, obj) {
-  return (obj || new NetEncoding.Float32Array).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * @param {number} index
- * @returns {number}
- */
-NetEncoding.Float32Array.prototype.data = function(index) {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? this.bb.readFloat32(this.bb.__vector(this.bb_pos + offset) + index * 4) : 0;
-};
-
-/**
- * @returns {number}
- */
-NetEncoding.Float32Array.prototype.dataLength = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
-};
-
-/**
- * @returns {Float32Array}
- */
-NetEncoding.Float32Array.prototype.dataArray = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? new Float32Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-NetEncoding.Float32Array.startFloat32Array = function(builder) {
-  builder.startObject(1);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} dataOffset
- */
-NetEncoding.Float32Array.addData = function(builder, dataOffset) {
-  builder.addFieldOffset(0, dataOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {Array.<number>} data
- * @returns {flatbuffers.Offset}
- */
-NetEncoding.Float32Array.createDataVector = function(builder, data) {
-  builder.startVector(4, data.length, 4);
-  for (var i = data.length - 1; i >= 0; i--) {
-    builder.addFloat32(data[i]);
-  }
-  return builder.endVector();
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {number} numElems
- */
-NetEncoding.Float32Array.startDataVector = function(builder, numElems) {
-  builder.startVector(4, numElems, 4);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-NetEncoding.Float32Array.endFloat32Array = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * @constructor
- */
-NetEncoding.Uint32Array = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {NetEncoding.Uint32Array}
- */
-NetEncoding.Uint32Array.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {NetEncoding.Uint32Array=} obj
- * @returns {NetEncoding.Uint32Array}
- */
-NetEncoding.Uint32Array.getRootAsUint32Array = function(bb, obj) {
-  return (obj || new NetEncoding.Uint32Array).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * @param {number} index
- * @returns {number}
- */
-NetEncoding.Uint32Array.prototype.data = function(index) {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? this.bb.readUint32(this.bb.__vector(this.bb_pos + offset) + index * 4) : 0;
-};
-
-/**
- * @returns {number}
- */
-NetEncoding.Uint32Array.prototype.dataLength = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
-};
-
-/**
- * @returns {Uint32Array}
- */
-NetEncoding.Uint32Array.prototype.dataArray = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? new Uint32Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-NetEncoding.Uint32Array.startUint32Array = function(builder) {
-  builder.startObject(1);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} dataOffset
- */
-NetEncoding.Uint32Array.addData = function(builder, dataOffset) {
-  builder.addFieldOffset(0, dataOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {Array.<number>} data
- * @returns {flatbuffers.Offset}
- */
-NetEncoding.Uint32Array.createDataVector = function(builder, data) {
-  builder.startVector(4, data.length, 4);
-  for (var i = data.length - 1; i >= 0; i--) {
-    builder.addInt32(data[i]);
-  }
-  return builder.endVector();
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {number} numElems
- */
-NetEncoding.Uint32Array.startDataVector = function(builder, numElems) {
-  builder.startVector(4, numElems, 4);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-NetEncoding.Uint32Array.endUint32Array = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * @constructor
- */
-NetEncoding.Int32Array = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {NetEncoding.Int32Array}
- */
-NetEncoding.Int32Array.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {NetEncoding.Int32Array=} obj
- * @returns {NetEncoding.Int32Array}
- */
-NetEncoding.Int32Array.getRootAsInt32Array = function(bb, obj) {
-  return (obj || new NetEncoding.Int32Array).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * @param {number} index
- * @returns {number}
- */
-NetEncoding.Int32Array.prototype.data = function(index) {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? this.bb.readInt32(this.bb.__vector(this.bb_pos + offset) + index * 4) : 0;
-};
-
-/**
- * @returns {number}
- */
-NetEncoding.Int32Array.prototype.dataLength = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
-};
-
-/**
- * @returns {Int32Array}
- */
-NetEncoding.Int32Array.prototype.dataArray = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? new Int32Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-NetEncoding.Int32Array.startInt32Array = function(builder) {
-  builder.startObject(1);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} dataOffset
- */
-NetEncoding.Int32Array.addData = function(builder, dataOffset) {
-  builder.addFieldOffset(0, dataOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {Array.<number>} data
- * @returns {flatbuffers.Offset}
- */
-NetEncoding.Int32Array.createDataVector = function(builder, data) {
-  builder.startVector(4, data.length, 4);
-  for (var i = data.length - 1; i >= 0; i--) {
-    builder.addInt32(data[i]);
-  }
-  return builder.endVector();
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {number} numElems
- */
-NetEncoding.Int32Array.startDataVector = function(builder, numElems) {
-  builder.startVector(4, numElems, 4);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-NetEncoding.Int32Array.endInt32Array = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * @constructor
- */
-NetEncoding.Float64Array = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {NetEncoding.Float64Array}
- */
-NetEncoding.Float64Array.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {NetEncoding.Float64Array=} obj
- * @returns {NetEncoding.Float64Array}
- */
-NetEncoding.Float64Array.getRootAsFloat64Array = function(bb, obj) {
-  return (obj || new NetEncoding.Float64Array).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * @param {number} index
- * @returns {number}
- */
-NetEncoding.Float64Array.prototype.data = function(index) {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? this.bb.readFloat64(this.bb.__vector(this.bb_pos + offset) + index * 8) : 0;
-};
-
-/**
- * @returns {number}
- */
-NetEncoding.Float64Array.prototype.dataLength = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
-};
-
-/**
- * @returns {Float64Array}
- */
-NetEncoding.Float64Array.prototype.dataArray = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? new Float64Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-NetEncoding.Float64Array.startFloat64Array = function(builder) {
-  builder.startObject(1);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} dataOffset
- */
-NetEncoding.Float64Array.addData = function(builder, dataOffset) {
-  builder.addFieldOffset(0, dataOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {Array.<number>} data
- * @returns {flatbuffers.Offset}
- */
-NetEncoding.Float64Array.createDataVector = function(builder, data) {
-  builder.startVector(8, data.length, 8);
-  for (var i = data.length - 1; i >= 0; i--) {
-    builder.addFloat64(data[i]);
-  }
-  return builder.endVector();
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {number} numElems
- */
-NetEncoding.Float64Array.startDataVector = function(builder, numElems) {
-  builder.startVector(8, numElems, 8);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-NetEncoding.Float64Array.endFloat64Array = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * @constructor
- */
-NetEncoding.JSONEncodedArray = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {NetEncoding.JSONEncodedArray}
- */
-NetEncoding.JSONEncodedArray.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {NetEncoding.JSONEncodedArray=} obj
- * @returns {NetEncoding.JSONEncodedArray}
- */
-NetEncoding.JSONEncodedArray.getRootAsJSONEncodedArray = function(bb, obj) {
-  return (obj || new NetEncoding.JSONEncodedArray).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * @param {number} index
- * @returns {number}
- */
-NetEncoding.JSONEncodedArray.prototype.data = function(index) {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? this.bb.readUint8(this.bb.__vector(this.bb_pos + offset) + index) : 0;
-};
-
-/**
- * @returns {number}
- */
-NetEncoding.JSONEncodedArray.prototype.dataLength = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
-};
-
-/**
- * @returns {Uint8Array}
- */
-NetEncoding.JSONEncodedArray.prototype.dataArray = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? new Uint8Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-NetEncoding.JSONEncodedArray.startJSONEncodedArray = function(builder) {
-  builder.startObject(1);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} dataOffset
- */
-NetEncoding.JSONEncodedArray.addData = function(builder, dataOffset) {
-  builder.addFieldOffset(0, dataOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {Array.<number>} data
- * @returns {flatbuffers.Offset}
- */
-NetEncoding.JSONEncodedArray.createDataVector = function(builder, data) {
-  builder.startVector(1, data.length, 1);
-  for (var i = data.length - 1; i >= 0; i--) {
-    builder.addInt8(data[i]);
-  }
-  return builder.endVector();
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {number} numElems
- */
-NetEncoding.JSONEncodedArray.startDataVector = function(builder, numElems) {
-  builder.startVector(1, numElems, 1);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-NetEncoding.JSONEncodedArray.endJSONEncodedArray = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * @constructor
- */
-NetEncoding.Column = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {NetEncoding.Column}
- */
-NetEncoding.Column.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {NetEncoding.Column=} obj
- * @returns {NetEncoding.Column}
- */
-NetEncoding.Column.getRootAsColumn = function(bb, obj) {
-  return (obj || new NetEncoding.Column).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * @returns {NetEncoding.TypedArray}
- */
-NetEncoding.Column.prototype.uType = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? /** @type {NetEncoding.TypedArray} */ (this.bb.readUint8(this.bb_pos + offset)) : NetEncoding.TypedArray.NONE;
-};
-
-/**
- * @param {flatbuffers.Table} obj
- * @returns {?flatbuffers.Table}
- */
-NetEncoding.Column.prototype.u = function(obj) {
-  var offset = this.bb.__offset(this.bb_pos, 6);
-  return offset ? this.bb.__union(obj, this.bb_pos + offset) : null;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-NetEncoding.Column.startColumn = function(builder) {
-  builder.startObject(2);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {NetEncoding.TypedArray} uType
- */
-NetEncoding.Column.addUType = function(builder, uType) {
-  builder.addFieldInt8(0, uType, NetEncoding.TypedArray.NONE);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} uOffset
- */
-NetEncoding.Column.addU = function(builder, uOffset) {
-  builder.addFieldOffset(1, uOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-NetEncoding.Column.endColumn = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * @constructor
- */
-NetEncoding.Matrix = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {NetEncoding.Matrix}
- */
-NetEncoding.Matrix.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {NetEncoding.Matrix=} obj
- * @returns {NetEncoding.Matrix}
- */
-NetEncoding.Matrix.getRootAsMatrix = function(bb, obj) {
-  return (obj || new NetEncoding.Matrix).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * @returns {number}
- */
-NetEncoding.Matrix.prototype.nRows = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? this.bb.readUint32(this.bb_pos + offset) : 0;
-};
-
-/**
- * @returns {number}
- */
-NetEncoding.Matrix.prototype.nCols = function() {
-  var offset = this.bb.__offset(this.bb_pos, 6);
-  return offset ? this.bb.readUint32(this.bb_pos + offset) : 0;
-};
-
-/**
- * @param {number} index
- * @param {NetEncoding.Column=} obj
- * @returns {NetEncoding.Column}
- */
-NetEncoding.Matrix.prototype.columns = function(index, obj) {
-  var offset = this.bb.__offset(this.bb_pos, 8);
-  return offset ? (obj || new NetEncoding.Column).__init(this.bb.__indirect(this.bb.__vector(this.bb_pos + offset) + index * 4), this.bb) : null;
-};
-
-/**
- * @returns {number}
- */
-NetEncoding.Matrix.prototype.columnsLength = function() {
-  var offset = this.bb.__offset(this.bb_pos, 8);
-  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
-};
-
-/**
- * @returns {NetEncoding.TypedArray}
- */
-NetEncoding.Matrix.prototype.colIndexType = function() {
-  var offset = this.bb.__offset(this.bb_pos, 10);
-  return offset ? /** @type {NetEncoding.TypedArray} */ (this.bb.readUint8(this.bb_pos + offset)) : NetEncoding.TypedArray.NONE;
-};
-
-/**
- * @param {flatbuffers.Table} obj
- * @returns {?flatbuffers.Table}
- */
-NetEncoding.Matrix.prototype.colIndex = function(obj) {
-  var offset = this.bb.__offset(this.bb_pos, 12);
-  return offset ? this.bb.__union(obj, this.bb_pos + offset) : null;
-};
-
-/**
- * @returns {NetEncoding.TypedArray}
- */
-NetEncoding.Matrix.prototype.rowIndexType = function() {
-  var offset = this.bb.__offset(this.bb_pos, 14);
-  return offset ? /** @type {NetEncoding.TypedArray} */ (this.bb.readUint8(this.bb_pos + offset)) : NetEncoding.TypedArray.NONE;
-};
-
-/**
- * @param {flatbuffers.Table} obj
- * @returns {?flatbuffers.Table}
- */
-NetEncoding.Matrix.prototype.rowIndex = function(obj) {
-  var offset = this.bb.__offset(this.bb_pos, 16);
-  return offset ? this.bb.__union(obj, this.bb_pos + offset) : null;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-NetEncoding.Matrix.startMatrix = function(builder) {
-  builder.startObject(7);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {number} nRows
- */
-NetEncoding.Matrix.addNRows = function(builder, nRows) {
-  builder.addFieldInt32(0, nRows, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {number} nCols
- */
-NetEncoding.Matrix.addNCols = function(builder, nCols) {
-  builder.addFieldInt32(1, nCols, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} columnsOffset
- */
-NetEncoding.Matrix.addColumns = function(builder, columnsOffset) {
-  builder.addFieldOffset(2, columnsOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {Array.<flatbuffers.Offset>} data
- * @returns {flatbuffers.Offset}
- */
-NetEncoding.Matrix.createColumnsVector = function(builder, data) {
-  builder.startVector(4, data.length, 4);
-  for (var i = data.length - 1; i >= 0; i--) {
-    builder.addOffset(data[i]);
-  }
-  return builder.endVector();
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {number} numElems
- */
-NetEncoding.Matrix.startColumnsVector = function(builder, numElems) {
-  builder.startVector(4, numElems, 4);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {NetEncoding.TypedArray} colIndexType
- */
-NetEncoding.Matrix.addColIndexType = function(builder, colIndexType) {
-  builder.addFieldInt8(3, colIndexType, NetEncoding.TypedArray.NONE);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} colIndexOffset
- */
-NetEncoding.Matrix.addColIndex = function(builder, colIndexOffset) {
-  builder.addFieldOffset(4, colIndexOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {NetEncoding.TypedArray} rowIndexType
- */
-NetEncoding.Matrix.addRowIndexType = function(builder, rowIndexType) {
-  builder.addFieldInt8(5, rowIndexType, NetEncoding.TypedArray.NONE);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} rowIndexOffset
- */
-NetEncoding.Matrix.addRowIndex = function(builder, rowIndexOffset) {
-  builder.addFieldOffset(6, rowIndexOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-NetEncoding.Matrix.endMatrix = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} offset
- */
-NetEncoding.Matrix.finishMatrixBuffer = function(builder, offset) {
-  builder.finish(offset);
-};
-
-// Exports for ECMAScript6 Modules
-export {NetEncoding};

client/src/util/stateManager/summarizeAnnotations.js

@@ -0,0 +1,102 @@
+import _ from "lodash";
+
+/*
+Build and return obs/var summary using any annotation in the schema
+
+Summary information for each annotation, keyed by annotation name.
+Value will be an object, containing summary information.
+
+For continuous annotations (int, float, etc):
+  <annotation_name>: {
+    range {
+      min: <number>,
+      max: <number>
+    }
+  }
+
+For categorical annotations (boolean, string, category):
+  <annotatoin_name>: {
+    options: {
+      <option1>: <number>,
+      ...
+    },
+    numOptions: <number>
+  }
+
+Summarize will be returned for BOTH obs and var annotations.
+
+Example:
+  {
+    "Splice_sites_Annotated": {
+      "range": {
+        "min": 26,
+        "max": 1075869
+      }
+    },
+    "Selection": {
+      numOptions, 6,
+      "options": {
+        "Astrocytes(HEPACAM)": 714,
+        "Endothelial(BSC)": 123,
+        "Oligodendrocytes(GC)": 294,
+        "Neurons(Thy1)": 685,
+        "Microglia(CD45)": 1108,
+        "Unpanned": 665
+      }
+    }
+  }
+
+NOTE: will not summarize the required 'name' annotation, as that is
+specified as unique per element.
+
+TODO: XXX - this data structure coerces all metadata categories into a string
+(ie, stores values as an Object property in the `options` field).   This looses
+information (eg, type) for category types which are not strings.  Consider an
+alterative data structure that does not use the object property for non-string
+data types (and does not use _.countBy to summarize).
+*/
+function summarizeDimension(schema, annotations) {
+  return _(schema)
+    .filter(v => v.name !== "name")
+    .keyBy("name")
+    .mapValues(anno => {
+      const { name, type } = anno;
+      const continuous = type === "int32" || type === "float32";
+
+      if (!continuous) {
+        const categories = _.uniq(_.flatMap(annotations, name));
+        const options = _.countBy(annotations, name);
+        const numOptions = _.size(options);
+        return {
+          numOptions,
+          options,
+          categories
+        };
+      }
+
+      if (continuous) {
+        let min = Number.POSITIVE_INFINITY;
+        let max = Number.NEGATIVE_INFINITY;
+        _.forEach(annotations, obs => {
+          const val = Number(obs[name]);
+          min = val < min ? val : min;
+          max = val > max ? val : max;
+        });
+        return { range: { min, max } };
+      }
+
+      throw new Error("incomprehensible schema");
+    })
+    .value();
+}
+
+export default function summarizeAnnotations(
+  schema,
+  obsAnnotations,
+  varAnnotations
+) {
+  return {
+    obs: summarizeDimension(schema.annotations.obs, obsAnnotations),
+    var: summarizeDimension(schema.annotations.var, varAnnotations)
+  };
+}

client/src/util/stateManager/universe.js

@@ -1,18 +1,23 @@
 // jshint esversion: 6
 
 import _ from "lodash";
-
+import * as kvCache from "./keyvalcache";
-import decodeMatrixFBS from "./matrix";
+import summarizeAnnotations from "./summarizeAnnotations";
-import * as Dataframe from "../dataframe";
-import fromEntries from "../fromEntries";
-import { isFpTypedArray } from "../typeHelpers";
 
 /*
 Private helper function - create and return a template Universe
 */
 function templateUniverse() {
   /* default universe template */
+
+  /* varDataCache config - see kvCache for semantics */
+  const VarDataCacheLowWatermark = 32; // cache element count
+  const VarDataCacheTTLMs = 1000; // min cache time in MS
+
   return {
+    api: null,
+    finalized: false, // XXX: may not be needed
+
     nObs: 0,
     nVar: 0,
     schema: {},
@@ -20,14 +25,21 @@ function templateUniverse() {
     /*
     Annotations
     */
-    obsAnnotations: Dataframe.Dataframe.empty(),
+    obsAnnotations: [] /* all obs annotations, by obs index */,
-    varAnnotations: Dataframe.Dataframe.empty(),
+    varAnnotations: [] /* all var annotations, by var index */,
-    obsLayout: Dataframe.Dataframe.empty(),
+    obsNameToIndexMap: {} /* reverse map 'name' to index */,
+    varNameToIndexMap: {} /* reverse map 'name' to index */,
+    summary: null /* derived data summaries XXX: consider exploding in place */,
+
+    obsLayout: { X: [], Y: [] } /* xy layout */,
 
     /*
-    Var data columns - subset of all
+    Cache of var data (expression), by var annotation name.   Data can be
+    accesses as a POJO, but if you want caching semantics, use the kvCache
+    API (eg., kvCache.get(), kvCache.set(), ...), which will maintain the
+    LRU semantics.
     */
-    varData: Dataframe.Dataframe.empty(null, new Dataframe.KeyIndex())
+    varDataCache: kvCache.create(VarDataCacheLowWatermark, VarDataCacheTTLMs)
   };
 }
 
@@ -39,94 +51,114 @@ These functions are used exclusively by the actions and reducers to
 build an internal POJO for use by the rendering components.
 */
 
-function promoteTypedArray(o) {
+/*
-  /*
+generate any client-side transformations or summarization that
-  Decide what internal data type to use for the data returned from 
+is independent of REST API response formats.
-  the server.
+*/
-
+function finalize(universe) {
-  TODO - future optimization: not all int32/uint32 data series require
+  /* A bit of sanity checking! */
-  promotion to float64.  We COULD simply look at the data to decide. 
+  const { nObs, nVar } = universe;
-  */
+  if (
-  if (isFpTypedArray(o) || Array.isArray(o)) return o;
+    nObs !== universe.obsAnnotations.length ||
-
+    nObs !== universe.obsLayout.X.length ||
-  let TyepdArrayCtor;
+    nObs !== universe.obsLayout.Y.length ||
-  switch (o.constructor) {
+    nVar !== universe.varAnnotations.length
-    case Int8Array:
+  ) {
-    case Uint8Array:
+    throw new Error("Universe dimensionality mismatch - failed to load");
-    case Uint8ClampedArray:
-    case Int16Array:
-    case Uint16Array:
-      TyepdArrayCtor = Float32Array;
-      break;
-
-    case Int32Array:
-    case Uint32Array:
-      TyepdArrayCtor = Float64Array;
-      break;
-
-    default:
-      throw new Error("Unexpected data type returned from server.");
   }
-  if (o.constructor === TyepdArrayCtor) return o;
+  // TODO: add more sanity checks, such as:
-  return new TyepdArrayCtor(o);
+  //  - all annotations in the schema
-}
+  //  - layout has supported number of dimensions
+  //  - ...
 
-function AnnotationsFBSToDataframe(arrayBuffer) {
   /*
-  Convert a Matrix FBS to a Dataframe.
+  Create all derived (convenience) data structures.
-
-  The application has strong assumptions that all scalar data will be
-  stored as a float32 or float64 (regardless of underlying data types).
-  For example, clipping of value ranges (eg, user-selected percentiles)
-  depends on the ability to use NaN in any numeric type.
-
-  All float data from the server is left as is.  All non-float is promoted
-  to an appropriate float.
   */
-  const fbs = decodeMatrixFBS(arrayBuffer, true); // leave in place
+  universe.obsNameToIndexMap = _.transform(
-  const columns = fbs.columns.map(c => {
+    universe.obsAnnotations,
-    if (isFpTypedArray(c) || Array.isArray(c)) return c;
+    (acc, value, idx) => {
-    return promoteTypedArray(c);
+      acc[value.name] = idx;
-  });
+    },
-  const df = new Dataframe.Dataframe(
+    {}
-    [fbs.nRows, fbs.nCols],
-    columns,
-    null,
-    new Dataframe.KeyIndex(fbs.colIdx)
   );
-  return df;
+  universe.varNameToIndexMap = _.transform(
+    universe.varAnnotations,
+    (acc, value, idx) => {
+      acc[value.name] = idx;
+    },
+    {}
+  );
+  universe.finalized = true;
+  return universe;
 }
 
-function LayoutFBSToDataframe(arrayBuffer) {
+function RESTv02AnnotationsResponseToInternal(response) {
-  const fbs = decodeMatrixFBS(arrayBuffer, true);
+  /*
-  if (fbs.columns.length < 2 || !fbs.columns.every(isFpTypedArray)) {
+  Source per the spec:
-    // We have strong assumptions about the shape & type of layout data.
+  {
-    throw new Error("Unexpected layout data type returned from server");
+    names: [
+      'tissue_type', 'sex', 'num_reads', 'clusters'
+    ],
+    data: [
+      [ 0, 'lung', 'F', 39844, 99 ],
+      [ 1, 'heart', 'M', 83, 1 ],
+      [ 49, 'spleen', null, 2, "unknown cluster" ],
+      // [ obsOrVarIndex, value, value, value, value ],
+      // ...
+    ]
   }
 
-  /*
+  Internal (target) format:
-  TODO: XXX
+  [
-
+    { __index__: 0, tissue_type: "lung", sex: "F", ... },
-  TEMPORARY CODE AND COMMENT to support the progressive implementation
+    ...
-  of multi-layout support.  For now, we search for one of the following 
+  ]
-  in the layouts and use it if we find it: umap, then tsne, then pca, 
-  then whatever is first in the list.
   */
-  let layoutIndex = 0;
+  const { names, data } = response;
-  ["umap", "tsne", "pca"].some(name => {
+  const keys = ["__index__", ...names];
-    const idx = fbs.colIdx.indexOf(`${name}_0`);
+  return _(data)
-    if (idx !== -1) {
+    .map(obs => _.zipObject(keys, obs))
-      layoutIndex = idx;
+    .value();
+}
+
+function RESTv02LayoutResponseToInternal(response) {
+  /*
+  Source per the spec:
+  {
+    layout: {
+      ndims: 2,
+      coordinates: [
+        [ 0, 0.284483, 0.983744 ],
+        [ 1, 0.038844, 0.739444 ],
+        // [ obsOrVarIndex, X_coord, Y_coord ],
+        // ...
+      ]
     }
-    return idx !== -1;
+  }
-  });
+
-  const df = new Dataframe.Dataframe(
+  Target (internal) format:
-    [fbs.nRows, 2],
+  {
-    [fbs.columns[layoutIndex], fbs.columns[layoutIndex + 1]],
+    X: Float32Array(numObs),
-    null,
+    Y: Float32Array(numObs)
-    new Dataframe.KeyIndex(["X", "Y"])
+  }
-  );
+  In the same order as obsAnnotations
-  return df;
+  */
+  const { ndims, coordinates } = response.layout;
+  if (ndims !== 2) {
+    throw new Error("Unsupported layout dimensionality");
+  }
+
+  const layout = {
+    X: new Float32Array(coordinates.length),
+    Y: new Float32Array(coordinates.length)
+  };
+
+  for (let i = 0; i < coordinates.length; i += 1) {
+    const [idx, x, y] = coordinates[i];
+    layout.X[idx] = x;
+    layout.Y[idx] = y;
+  }
+  return layout;
 }
 
 function reconcileSchemaCategoriesWithSummary(universe) {
@@ -140,27 +172,27 @@ function reconcileSchemaCategoriesWithSummary(universe) {
   cases, add a 'categories' field to the schema so it is accessible.
   */
 
-  universe.schema.annotations.obs.forEach(s => {
+  _.forEach(universe.schema.annotations.obs, s => {
     if (
       s.type === "string" ||
       s.type === "boolean" ||
       s.type === "categorical"
     ) {
       const categories = _.union(
-        s.categories ?? [],
+        _.get(s, "categories", []),
-        universe.obsAnnotations.col(s.name).summarize().categories ?? []
+        _.get(universe.summary.obs[s.name], "categories", [])
       );
       s.categories = categories;
     }
   });
 }
 
-export function createUniverseFromResponse(
+export function createUniverseFromRestV02Response(
   configResponse,
   schemaResponse,
   annotationsObsResponse,
   annotationsVarResponse,
-  layoutFBSResponse
+  layoutObsResponse
 ) {
   /*
   build & return universe from a REST 0.2 /config, /schema and /annotations/obs response
@@ -168,61 +200,61 @@ export function createUniverseFromResponse(
   const { schema } = schemaResponse;
   const universe = templateUniverse();
 
+  /* constants */
+  universe.api = "0.2";
+
   /* schema related */
   universe.schema = schema;
   universe.nObs = schema.dataframe.nObs;
   universe.nVar = schema.dataframe.nVar;
 
   /* annotations */
-  universe.obsAnnotations = AnnotationsFBSToDataframe(annotationsObsResponse);
+  universe.obsAnnotations = RESTv02AnnotationsResponseToInternal(
-  universe.varAnnotations = AnnotationsFBSToDataframe(annotationsVarResponse);
+    annotationsObsResponse
-  /* layout */
+  );
-  universe.obsLayout = LayoutFBSToDataframe(layoutFBSResponse);
+  universe.varAnnotations = RESTv02AnnotationsResponseToInternal(
+    annotationsVarResponse
+  );
 
-  /* sanity check */
+  /* layout */
-  if (
+  universe.obsLayout = RESTv02LayoutResponseToInternal(layoutObsResponse);
-    universe.nObs !== universe.obsLayout.length ||
+
-    universe.nObs !== universe.obsAnnotations.length ||
+  universe.summary = summarizeAnnotations(
-    universe.nVar !== universe.varAnnotations.length
+    universe.schema,
-  ) {
+    universe.obsAnnotations,
-    throw new Error("Universe dimensionality mismatch - failed to load");
+    universe.varAnnotations
-  }
+  );
 
   reconcileSchemaCategoriesWithSummary(universe);
-
+  return finalize(universe);
-  /* Index schema for ease of use */
-  universe.schema.annotations.obsByName = fromEntries(
-    universe.schema.annotations.obs.map(v => [v.name, v])
-  );
-  universe.schema.annotations.varByName = fromEntries(
-    universe.schema.annotations.var.map(v => [v.name, v])
-  );
-  return universe;
 }
 
-export function convertDataFBStoObject(universe, arrayBuffer) {
+export function convertExpressionRESTv02ToObject(universe, response) {
   /*
-  /data/var returns a flatbuffer (FBS) as described by cellxgene/fbs/matrix.fbs
+  /data/obs response looks like:
-
-  This routine converts the binary wire encoding into a JS object:
-
   {
-    gene: Float32Array,
+    var: [ varIndices fetched ],
-    ...
+    obs: [
+      [ obsIndex, evalue, ... ],
+      ...
+    ]
   }
+
+  convert expression toa simple Float32Array, and return
+  { geneName: array, geneName: array, ... }
+  NOTE: geneName, not varIndex
   */
-  const fbs = decodeMatrixFBS(arrayBuffer);
+  const vars = response.var;
-  const { colIdx, columns } = fbs;
+  const { obs } = response;
   const result = {};
-
+  // XXX TODO: could this use _.unzip and have less code?
-  if (!columns.every(isFpTypedArray)) {
+  for (let varIdx = 0; varIdx < vars.length; varIdx += 1) {
-    // We have strong assumptions that all var data is float
+    const gene = universe.varAnnotations[vars[varIdx]].name;
-    throw new Error("Unexpected non-floating point response from server.");
+    const data = new Float32Array(universe.nObs);
-  }
+    for (let obsIdx = 0; obsIdx < obs.length; obsIdx += 1) {
-
+      data[obsIdx] = obs[obsIdx][varIdx + 1];
-  for (let c = 0; c < colIdx.length; c += 1) {
+    }
-    const varName = universe.varAnnotations.at(colIdx[c], "name");
+    result[gene] = data;
-    result[varName] = columns[c];
   }
   return result;
 }

client/src/util/stateManager/world.js

@@ -1,17 +1,12 @@
 // jshint esversion: 6
 
-import clip from "../clip";
+import _ from "lodash";
-import {
+import * as kvCache from "./keyvalcache";
-  layoutDimensionName,
+import summarizeAnnotations from "./summarizeAnnotations";
-  obsAnnoDimensionName,
+import { layoutDimensionName, obsAnnoDimensionName } from "../nameCreators";
-  diffexpDimensionName,
+import { sliceByIndex } from "../typedCrossfilter/util";
-  userDefinedDimensionName
-} from "../nameCreators";
-import * as Dataframe from "../dataframe";
-import ImmutableTypedCrossfilter from "../typedCrossfilter/crossfilter";
 
 /*
-
 World is a subset of universe.   Most code should use world, and should
 (generally) not use Universe.   World contains any per-obs or per-var data
 that must be consistent acorss the app when we view/manipulate subsets
@@ -20,218 +15,176 @@ of Universe.
 Private API indicated by leading underscore in key name (eg, _foo).  Anything else
 is public.
 
-Notable keys in the world object:
+World contains several public keys, obsAnnotations, and obsLayout, which are
-
+arrays contianing information about an OBS in the same order/offset.  In
-* nObs, nVar: dimensions
+other words, world.obsAnnotations[0] and world.obsLayout.X[0] refer to the same
-
+obs/cell.
-* schema: data schema from the server
-
-* clipQuantiles: the quantiles used to clip all data in world.
 
 * obsAnnotations:
 
-  Dataframe containing obs annotations.  Columns are indexed by annotation
+  obsAnnotations will return an array of objects.  Each object contains all annotation
-  name (eg, 'tissue type'), and rows are indexed by the REST API obsIndex
+  values for a given observation/cell, keyed by annotation name, PLUS a key
-  (ie, the offset into the underlying server-side dataframe).
+  '__cellId__', containing a REST API ID for this obs/cell (referred to as the
+  obsIndex in the REST 0.2 spec or cellIndex in the 0.1 spec.
 
-  This indexing means that you can access data by _either_ the server's
+  Example:  [ { __cellId__: 99, cluster: 'blue', numReads: 93933 } ]
-  obxIndex, or the offset into the client-side column array .  Be careful
+
-  to know which you want and are using.
+  NOTE: world.obsAnnotation should be identical to the old state.cells value,
+  EXCEPT that
+    * __cellIndex__ renamed to __index__
+    * __x__ and __y__ are now in world.obsLayout
+    * __color__ and __colorRBG__ should be moved to controls reducer
 
 * obsLayout:
 
-  A dataframe containing the X/Y layout for all obs.  Columns are named
+  obsLayout will return an object containing two arrays, containing X and Y
-  'X' and 'Y', and rows are indexed in the same way as obsAnnotation.
+  coordinates respectively.
 
-* varData: a cache of expression columns, stored in a Dataframe.  Cache
+  Example: { X: [ 0.33, 0.23, ... ], Y: [ 0.8, 0.777, ... ]}
-  managed by controls reducer.
 
-* unclipped: will contain unclipped variants of all potentiall clipped
+* crossfilter - a crossfilter object across world.obsAnnotations
-  dataframes (obsAnnotations, varData).
+
+* dimensionMap - an object mapping annotation names to dimensions on
+  the crossfilter
 
 */
 
+/* varDataCache config - see kvCache for semantics */
+const VarDataCacheLowWatermark = 32; // cache element count
+const VarDataCacheTTLMs = 1000; // min cache time in MS
+
 function templateWorld() {
-  const obsAnnotations = Dataframe.Dataframe.empty();
-  const varAnnotations = Dataframe.Dataframe.empty();
-  const obsLayout = Dataframe.Dataframe.empty();
-  const varData = Dataframe.Dataframe.empty(null, new Dataframe.KeyIndex());
   return {
+    // map from universe obsIndex to world offset.
+    // Undefined / null indicates identity mapping.
+    obsIndex: null,
+    obsBackIndex: null,
+
     /* schema/version related */
+    api: null,
     schema: null,
     nObs: 0,
     nVar: 0,
-    clipQuantiles: { min: 0, max: 1 },
 
     /* annotations */
-    obsAnnotations,
+    obsAnnotations: null,
-    varAnnotations,
+    varAnnotations: null,
 
-    /* layout of graph. Dataframe. */
+    /* layout of graph */
-    obsLayout,
+    obsLayout: null,
 
-    /* Var data columns - subset of all data (may be empty) */
+    /* derived data summaries XXX: consider exploding in place */
-    varData,
+    summary: null,
 
-    /* unclipped dataframes - subset, but not value clipped */
+    varDataCache: kvCache.create(
-    unclipped: {
+      VarDataCacheLowWatermark,
-      obsAnnotations,
+      VarDataCacheTTLMs
-      varData
+    ) /* cache of var data (expression) */
-    }
   };
 }
 
-function clipDataframe(
-  df,
-  lowerQuantile,
-  upperQuantile,
-  quantileF,
-  clipPredicate = () => true,
-  value = Number.NaN
-) {
-  /*
-  For all columns in the dataframe, clip all values above or below specified 
-  quantiles to `value` if clipPredicate returns True for that column (if it
-  returns false, skip the column entirely).
-
-  Returns a clipped copy - does not mutate original.
-
-  clipPredicate must have signature: (dataframe, colIndex, colLabel) => boolean
-  True signifies that the column should be clipped; false indicates that the
-  column should be left intact/unchanged.
-
-  quantileF must have signature:  (label, qval) => number
-  */
-  if (lowerQuantile < 0) lowerQuantile = 0;
-  if (upperQuantile > 1) upperQuantile = 1;
-  if (lowerQuantile === 0 && upperQuantile === 1) return df;
-
-  const keys = df.colIndex.keys();
-  return df.mapColumns((col, colIdx) => {
-    const colLabel = keys[colIdx];
-    if (!clipPredicate(df, colIdx, colLabel)) return col;
-
-    const colMin = quantileF(colLabel, lowerQuantile);
-    const colMax = quantileF(colLabel, upperQuantile);
-    const newCol = clip(col.slice(), colMin, colMax, value);
-    return newCol;
-  });
-}
-
-/*
-Create World with contents eq entire universe.   Commonly used to initialize World.
-If clipQuantiles
-*/
 export function createWorldFromEntireUniverse(universe) {
+  if (!universe.finalized) {
+    throw new Error("World can't be created from an partial Universe");
+  }
+
   const world = templateWorld();
 
+  // map from the universe obsIndex to our world offset.
+  // undefined/null indicates identity map.
+  // In other words   obsBackIndex[universeIdx] -> worldIdx
+  world.obsBackIndex = null;
+  // Map to the universe index for each element in world.
+  // Null indicates identity map (aka world === universe)
+  // In other wrods  obsIndex[worldIdx]  ->  universeIdx
+  world.obsIndex = null;
+
+  /*
+  public interface follows
+  */
+
   /* Schema related */
+  world.api = universe.api;
   world.schema = universe.schema;
   world.nObs = universe.nObs;
   world.nVar = universe.nVar;
-  world.clipQuantiles = { min: 0, max: 1 };
 
-  /* dataframes: annotations and layout */
+  /* annotations */
-  world.obsAnnotations = universe.obsAnnotations.clone();
+  world.obsAnnotations = universe.obsAnnotations;
-  world.varAnnotations = universe.varAnnotations.clone();
+  world.varAnnotations = universe.varAnnotations;
-  world.obsLayout = universe.obsLayout.clone();
 
-  /* Var dataframe - contains a subset of all var columns */
+  /* layout and display characteristics */
-  world.varData = universe.varData.clone();
+  world.obsLayout = universe.obsLayout;
 
-  /* save unclipped copies of potentially clipped dataframes */
+  /* derived data & summaries */
-  world.unclipped = {
+  world.summary = summarizeAnnotations(
-    obsAnnotations: world.obsAnnotations.clone(),
+    world.schema,
-    varData: world.varData.clone()
+    world.obsAnnotations,
-  };
+    world.varAnnotations
+  );
+
+  /* build the varDataCache */
+  world.varDataCache = kvCache.map(
+    universe.varDataCache,
+    val => subsetVarData(world, universe, val),
+    { lowWatermark: VarDataCacheLowWatermark, minTTL: VarDataCacheTTLMs }
+  );
 
   return world;
 }
 
-/*
+export function createWorldFromCurrentSelection(universe, world, crossfilter) {
-clip dataframes based on quantiles.
+  const newWorld = templateWorld();
 
-This is an in-place operation on the world object provided as an argument.
+  /* these don't change as only OBS are selected in our current implementation */
-The values in world.unclipped are clipped and assigned to world.obsAnnotations
+  newWorld.api = universe.api;
-and world.varData.
+  newWorld.nVar = universe.nVar;
-*/
+  newWorld.schema = universe.schema;
-function setClippedDataframes(world) {
+  newWorld.varAnnotations = universe.varAnnotations;
-  const { schema } = world;
-  const isContinuousObsAnnotation = (df, idx, label) =>
-    deduceDimensionType(schema.annotations.obsByName[label], label) !== "enum";
-  const obsQuantile = (label, q) =>
-    world.unclipped.obsAnnotations.col(label).summarize().percentiles[100 * q];
-  world.obsAnnotations = clipDataframe(
-    world.unclipped.obsAnnotations,
-    world.clipQuantiles.min,
-    world.clipQuantiles.max,
-    obsQuantile,
-    isContinuousObsAnnotation
-  );
 
-  const varDataQuantile = (label, q) =>
+  /* build index maps and back maps based upon current selection state */
-    world.unclipped.varData.col(label).summarize().percentiles[100 * q];
+  const obsBackIndex = new Uint32Array(universe.nObs);
-  world.varData = clipDataframe(
+  obsBackIndex.fill(-1); // default - aka unused
-    world.unclipped.varData,
+  const notSelected = obsBackIndex[0];
-    world.clipQuantiles.min,
+  let nObs = 0;
-    world.clipQuantiles.max,
+  for (let i = 0; i < universe.nObs; i += 1) {
-    varDataQuantile,
+    if (crossfilter.isElementFiltered(i)) {
-    () => true
+      obsBackIndex[i] = nObs;
-  );
+      nObs += 1;
-}
+    }
-
+  }
-/*
+  const obsIndex = new Uint32Array(nObs);
-Subset the current world based upon the current selection, maintaining any existing
+  for (let i = 0; i < universe.nObs; i += 1) {
-clip.  Returns new world.  Parameters:
+    const worldIdx = obsBackIndex[i];
-  * unvierse
+    if (worldIdx !== notSelected) {
-  * world - the current world
+      obsIndex[worldIdx] = i;
-  * crossfilter - the selection state
+    }
-*/
-export function createWorldBySelection(universe, world, crossfilter) {
-  const newWorld = { ...world, obsLayout: null, unclipped: {}, varData: null };
-
-  /* subset unclipped dataframes based upon current selection */
-  const mask = crossfilter.allSelectedMask();
-  newWorld.obsLayout = world.obsLayout.isubsetMask(mask);
-  newWorld.unclipped.obsAnnotations = world.unclipped.obsAnnotations.isubsetMask(
-    mask
-  );
-  if (world.unclipped.varData.isEmpty()) {
-    newWorld.unclipped.varData = world.unclipped.varData.clone();
-  } else {
-    newWorld.unclipped.varData = world.unclipped.varData.isubsetMask(mask);
   }
-  /* subsetting changings dimension size */
-  newWorld.nObs = newWorld.unclipped.obsAnnotations.dims[0];
 
-  /* and now clip */
+  newWorld.nObs = nObs;
-  setClippedDataframes(newWorld);
+  newWorld.obsIndex = obsIndex;
-  return newWorld;
+  newWorld.obsBackIndex = obsBackIndex;
-}
 
-/*
+  /* now slice */
-Change clip quantiles on the current world, returning a new world.
+  newWorld.obsAnnotations = sliceByIndex(universe.obsAnnotations, obsIndex);
-Parameters:
+  newWorld.obsLayout = {
-  * universe
+    X: sliceByIndex(universe.obsLayout.X, obsIndex),
-  * world - current world
+    Y: sliceByIndex(universe.obsLayout.Y, obsIndex)
-  * clipQuantiles - new clip
-*/
-export function createWorldWithNewClip(
-  universe,
-  world,
-  crossfilter,
-  clipQuantiles
-) {
-  const newWorld = { ...world, obsAnnotation: null, varData: null };
-  newWorld.clipQuantiles = clipQuantiles;
-  newWorld.obsLayout = world.obsLayout.clone();
-  newWorld.unclipped = {
-    obsAnnotations: world.unclipped.obsAnnotations.clone(),
-    varData: world.unclipped.varData.clone()
   };
 
-  /* and now clip */
+  /* derived data & summaries */
-  setClippedDataframes(newWorld);
+  newWorld.summary = summarizeAnnotations(
+    newWorld.schema,
+    newWorld.obsAnnotations,
+    newWorld.varAnnotations
+  );
+
+  /* build the varDataCache */
+  newWorld.varDataCache = kvCache.map(
+    universe.varDataCache,
+    val => subsetVarData(newWorld, universe, val),
+    { lowWatermark: VarDataCacheLowWatermark, minTTL: VarDataCacheTTLMs }
+  );
   return newWorld;
 }
 
@@ -260,55 +213,72 @@ function deduceDimensionType(attributes, fieldName) {
   return dimensionType;
 }
 
-export function createObsDimensions(crossfilter, world) {
+/*
-  /*
+  Return a crossfilter dimension for the specified world & named gene.
-  create and return a crossfilter with a dimension for every obs annotation
-  for which we have a supported type, *except* 'name'
-  */
-  const { schema, obsLayout, obsAnnotations } = world;
-  const annoList = schema.annotations.obs.filter(anno => anno.name !== "name");
-  crossfilter = annoList.reduce((xfltr, anno) => {
-    const dimType = deduceDimensionType(anno, anno.name);
-    const colData = obsAnnotations.col(anno.name).asArray();
-    const name = obsAnnoDimensionName(anno.name);
-    if (dimType === "enum") {
-      return xfltr.addDimension(name, "enum", colData);
-    }
-    if (dimType) {
-      return xfltr.addDimension(name, "scalar", colData, dimType);
-    }
-    return xfltr;
-  }, crossfilter);
 
-  return crossfilter.addDimension(
+  NOTE: this assumes that the expression data was already loaded,
-    layoutDimensionName("XY"),
+  by calling an appropriate action creator.
-    "spatial",
+
-    obsLayout.col("X").asArray(),
+  Caller needs to *save* this dimension somewhere for it to be later used.
-    obsLayout.col("Y").asArray()
+  Dimension must be destroyed by calling dimension.dispose()
+  when it is no longer needed
+  (it will not be garbage collected without this call)
+*/
+
+export function createVarDimension(
+  world,
+  _worldVarDataCache,
+  crossfilter,
+  geneName
+) {
+  return crossfilter.dimension(_worldVarDataCache[geneName], Float32Array);
+}
+
+export function createObsDimensionMap(crossfilter, world) {
+  /*
+  create and return a crossfilter dimension for every obs annotation
+  for which we have a supported type.
+  */
+  const { schema, obsLayout } = world;
+
+  // Create a crossfilter dimension for all obs annotations *except* 'name'
+  const dimensionMap = _(schema.annotations.obs)
+    .filter(anno => anno.name !== "name")
+    .transform((result, anno) => {
+      const dimType = deduceDimensionType(anno, anno.name);
+      // XXX if dimtype is a scalar, we may be able to do better?
+      if (dimType) {
+        result[obsAnnoDimensionName(anno.name)] = crossfilter.dimension(
+          r => r[anno.name],
+          dimType
+        );
+      } // else ignore the annotation
+    }, {})
+    .value();
+
+  /*
+  Add crossfilter dimensions allowing filtering on layout
+  */
+  dimensionMap[layoutDimensionName("X")] = crossfilter.dimension(
+    obsLayout.X,
+    Float32Array
   );
+  dimensionMap[layoutDimensionName("Y")] = crossfilter.dimension(
+    obsLayout.Y,
+    Float32Array
+  );
+
+  return dimensionMap;
 }
 
 export function worldEqUniverse(world, universe) {
-  return (
+  return world.obsAnnotations === universe.obsAnnotations;
-    world.obsAnnotations === universe.obsAnnotations ||
-    world.obsAnnotations.rowIndex === universe.obsAnnotations.rowIndex
-  );
 }
 
-export function getSelectedByIndex(crossfilter) {
+export function subsetVarData(world, universe, varData) {
-  /*
+  // If world === universe, just return the entire varData array
-  return array of obsIndex, containing all selected obs/cells.
+  if (worldEqUniverse(world, universe)) {
-  */
+    return varData;
-  const selected = crossfilter.allSelectedMask(); // array of bool-ish
-  const keys = crossfilter.data.rowIndex.keys(); // row keys, aka universe rowIndex
-
-  const set = new Int32Array(selected.length);
-  let numElems = 0;
-  for (let i = 0, l = selected.length; i < l; i += 1) {
-    if (selected[i]) {
-      set[numElems] = keys[i];
-      numElems += 1;
-    }
   }
-  return new Int32Array(set.buffer, 0, numElems);
+  return sliceByIndex(varData, world.obsIndex);
 }

client/src/util/stateManager/worldUtil.js

@@ -1,75 +0,0 @@
-/* eslint-disable import/prefer-default-export */
-import _ from "lodash";
-
-/*
-Various utility functions operating on World/Universe
-*/
-
-/*
-Count unique category values, binning first by dim1 then by dim2
-Return:
-
-Map {
-   dim1_val1: Map {
-      dim2_val1: number,
-      dim2_val2: number,
-      ...
-   },
-   ...
-}
-
-Parameters are:
-- dim1: dimension 1 name/label
-- dim2: dimension 2 name/label
-- df: dataframe containing dim1 and dim2 on the column axis
-
-*/
-function _countCategoryValues2D(dim1, dim2, df) {
-  const dimMap = new Map();
-  const col1 = df.col(dim1) ? df.col(dim1).asArray() : null;
-  const col2 = df.col(dim2) ? df.col(dim2).asArray() : null;
-  if (!col1 || !col2) {
-    return dimMap;
-  }
-
-  for (let r = 0, l = df.length; r < l; r += 1) {
-    const val1 = col1[r];
-    const val2 = col2[r];
-    let d2Map = dimMap.get(val1);
-    if (d2Map === undefined) {
-      d2Map = new Map();
-      dimMap.set(val1, d2Map);
-    }
-    let curCount = d2Map.get(val2);
-    if (curCount === undefined) {
-      curCount = 0;
-    }
-    d2Map.set(val2, curCount + 1);
-  }
-  return dimMap;
-}
-
-let __worldUtilMemoId__ = 0;
-function _memoizedId(x) {
-  if (!x.__worldUtilMemoId__) {
-    __worldUtilMemoId__ += 1;
-    x.__worldUtilMemoId__ = __worldUtilMemoId__;
-  }
-  return x.__worldUtilMemoId__;
-}
-function _countCategoryValues2DResolver(...args) {
-  const id = args[0] + args[1] + _memoizedId(args[2]);
-  return id;
-}
-
-export const countCategoryValues2D = _.memoize(
-  _countCategoryValues2D,
-  _countCategoryValues2DResolver
-);
-
-/*
-Clear any cached data within WorldUtil caches, eg, memoized functions
-*/
-export function clearCaches() {
-  countCategoryValues2D.cache.clear();
-}

client/src/util/statemachine/index.js

@@ -1,95 +0,0 @@
-/*
-Very simple FSM for use in reducer, etc.
-
-To create a state machine:
-  new StateMachine(initialState, transitions, onErrorCallback) -> statemachine
-
-Where:
-  * initialState - a caller-specified value that represents the initial state of
-    the FSM.
-  * transitions - an array of objects, representing FSM transitions (graph edges),
-    having the form:
-      {
-        to: state_name_transitioning_to,
-        from: state_name_transitioning_from,
-        event: value_that_will_cause_transition,
-        action: optional_callback_upon_transition
-      }
-    The transition will be provided to the action callback, so other data
-    may be stored in the transition object for use by the action callback.
-  * onErrorCallback - a callback function called if the FSM receives an event
-    for which it has no defined transition.
-
-
-Interface:
-  * states - property containing the state names. A Set(), contianing the
-    union of to: and from: values.
-  * events - property containing all of the accepted event values.  Set().
-  * graph - a Map of Maps, organized as  graph[eventValue][fromStateValue]
-  * clone() - clone the entire statemachine.
-  * next(eventValue) - drive the FSM to the next state.   If the event
-    matches a transition with a defined action, the action callback is
-    called, and the action return value is returned by next().  If no
-    transition is defined, onErrorCallback is called.
-
-Example:
-
-  const transitions = [
-    { from: "A", to: "B", event: "yo", action: () => 42 }
-  ];
-  const fsm = new StateMachine("A", transitions, () => { throw new Error("oops") });
-  fsm.next("yo");   // returns 42
-
-
-*/
-export default class StateMachine {
-  constructor(initState, transitions, onError) {
-    this.onError = onError || (() => undefined);
-    this.state = initState;
-
-    // all states
-    this.states = new Set(
-      transitions.reduce((names, tsn) => {
-        names.push(tsn.from);
-        names.push(tsn.to);
-        return names;
-      }, [])
-    );
-
-    // all transition names (aka events)
-    this.events = new Set(transitions.map(tsn => tsn.event));
-
-    // the transition graph.
-    // graph[event][from] -> transition
-    this.graph = transitions.reduce((graph, tsn) => {
-      const { event, from } = tsn;
-      if (!graph.has(event)) graph.set(event, new Map());
-      const tsnMap = graph.get(event);
-      tsnMap.set(from, tsn);
-      return graph;
-    }, new Map());
-  }
-
-  clone(initState) {
-    const fsm = new StateMachine(initState, []);
-    fsm.onError = this.onError;
-    fsm.states = this.states;
-    fsm.events = this.events;
-    fsm.graph = this.graph;
-    return fsm;
-  }
-
-  next(event, data) {
-    const { graph, state } = this;
-    const tsnMap = graph.get(event);
-    if (!tsnMap) return this.onError(this, event, state, undefined);
-
-    const transition = tsnMap.get(state);
-    if (!transition) return this.onError(this, event, state, undefined);
-
-    this.state = transition.to;
-    return transition.action
-      ? transition.action(this, transition, data)
-      : undefined;
-  }
-}

client/src/util/typeHelpers.js

@@ -1,29 +0,0 @@
-/*
-Various type and schema related helper functions.
-*/
-
-/*
-Utility function to test for a typed array
-*/
-export function isTypedArray(x) {
-	return (
-		ArrayBuffer.isView(x) &&
-		Object.prototype.toString.call(x) !== "[object DataView]"
-	);
-}
-
-/*
-Test for float typed array, ie, Float32TypedArray or Float64TypedArray
-*/
-export function isFpTypedArray(x) {
-	let constructor;
-	const isFloatArray =
-		x &&
-		({ constructor } = x) &&
-		(constructor === Float32Array || constructor === Float64Array);
-	return isFloatArray;
-}
-
-export function isArrayOrTypedArray(x) {
-	return Array.isArray(x) || isTypedArray(x);
-}

client/src/util/typedCrossfilter/bitArray.js

@@ -40,7 +40,7 @@ class BitArray {
   // Return the number of records that are selected, ie, have a one bit in
   // all allocated dimensions.
   //
-  selectionCount() {
+  get selectionCount() {
     return this.countAllOnes();
   }
 
@@ -48,27 +48,16 @@ class BitArray {
   //
   countAllOnes() {
     let count = 0;
-    const { bitarray, length, width } = this;
+    const { bitarray, bitmask, length, width } = this;
-    if (width === 1) {
+    for (let l = 0; l < length; l += 1) {
-      // special case, width === 1, for performance
+      let dimensionsSet = 0;
-      const bitmask = this.bitmask[0];
+      for (let w = 0; w < width; w += 1) {
-      for (let l = 0; l < length; l += 1) {
+        if (bitarray[w * length + l] === bitmask[w]) {
-        if (bitarray[l] === bitmask) {
+          dimensionsSet += 1;
-          count += 1;
         }
       }
-    } else {
+      if (dimensionsSet === width) {
-      const { bitmask } = this;
+        count += 1;
-      for (let l = 0; l < length; l += 1) {
-        let dimensionsSet = 0;
-        for (let w = 0; w < width; w += 1) {
-          if (bitarray[w * length + l] === bitmask[w]) {
-            dimensionsSet += 1;
-          }
-        }
-        if (dimensionsSet === width) {
-          count += 1;
-        }
       }
     }
     return count;
@@ -211,19 +200,6 @@ class BitArray {
     }
   }
 
-  // select range of indices on a dimension
-  //
-  selectFromRange(dim, range) {
-    const col = dim >>> 5;
-    const first = range[0];
-    const last = range[1];
-    const one = 1 << dim % 32;
-    const offset = col * this.length;
-    for (let i = first; i < last; i += 1) {
-      this.bitarray[offset + i] |= one;
-    }
-  }
-
   // select range of indices on a dimension, indirect through a sort map.
   // Indirect functions are used to map between sort and natural order.
   //
@@ -238,19 +214,6 @@ class BitArray {
     }
   }
 
-  // deselect range of indices on a dimension
-  //
-  deselectFromRange(dim, range) {
-    const col = dim >>> 5;
-    const first = range[0];
-    const last = range[1];
-    const zero = ~(1 << dim % 32);
-    const offset = col * this.length;
-    for (let i = first; i < last; i += 1) {
-      this.bitarray[offset + i] &= zero;
-    }
-  }
-
   // deselect range of indices on a dimension, indirect through a sort map.
   //
   deselectIndirectFromRange(dim, indirect, range) {
@@ -270,14 +233,12 @@ class BitArray {
   fillBySelection(result, selectedValue, deselectedValue) {
     // special case (width === 1) for performance
     if (this.width === 1) {
-      const { bitmask, bitarray } = this;
+      const bitmask = this.bitmask[0];
-      const mask = bitmask[0];
+      for (let i = 0, len = this.length; i < len; i += 1) {
-      if (!mask) {
+        result[i] =
-        result.fill(deselectedValue);
+          bitmask && this.bitarray[i] === bitmask
-      } else {
+            ? selectedValue
-        for (let i = 0, len = this.length; i < len; i += 1) {
+            : deselectedValue;
-          result[i] = bitarray[i] === mask ? selectedValue : deselectedValue;
-        }
       }
     } else {
       for (let i = 0, len = this.length; i < len; i += 1) {