Surround script hash in single quotes (#1735)

* add disclaimer about changing the script

* add hash for obsolete browser script

* add explicit domain img-src

* add single quotes

* add quotes to `data:`

* change order and remove single quotes

* lint, remove extra slash, and make hash an array

* add links to S3

* surround script has in single quotes

.bumpversion.cfg

@@ -0,0 +1,14 @@
+[bumpversion]
+current_version = 0.16.0
+
+[bumpversion:file:setup.py]
+search = version="{current_version}"
+replace = version="{new_version}"
+
+[bumpversion:file:client/package.json]
+search = "version": "{current_version}"
+replace = "version": "{new_version}"
+
+[bumpversion:file:server/__init__.py]
+search = __version__ = "{current_version}"
+replace = __version__ = "{new_version}"

.codecov.yml

@@ -0,0 +1,8 @@
+coverage:
+  status:
+    project:
+      default:
+        informational: true
+    patch:
+      default:
+        informational: true

.dockerignore

@@ -0,0 +1,5 @@
+bin
+client
+dist
+docs
+server

.github/ISSUE_TEMPLATE/---bug-report.md

@@ -0,0 +1,32 @@
+---
+name: "\U0001F41E Bug report"
+about: Create a report to identify a bug in cellxgene
+title: "[BUG]"
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Version (please complete the following information):**
+ - Desktop or hosted?:
+ - Browser (if hosted) [e.g. chrome, safari]:
+ - Version [e.g. 0.13.0]:
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/---feature-request.md

@@ -0,0 +1,20 @@
+---
+name: "\U0001F4A1 Feature request"
+about: Suggest an idea for this project
+title: "[FEATURE REQUEST]"
+labels: user request
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/---question-clarification.md

@@ -0,0 +1,10 @@
+---
+name: "\U0001F9D0 Question/Clarification"
+about: Ask a question or for a clarification
+title: "[QUESTION]"
+labels: question
+assignees: ''
+
+---
+
+

.github/workflows/compatibility_tests.yml

@@ -0,0 +1,118 @@
+name: Compatibility Tests
+
+on:
+  schedule:
+    - cron: '0 8 7 * 2'
+  push:
+    branches:
+      - main
+
+env:
+  JEST_ENV: prod
+
+jobs:
+  docker-build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Build docker image
+        run: docker build .
+
+  cellxgene-main-with-python-and-anndata-versions:
+    name: python versions x anndata versions
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8]
+        anndata-version: [0.6.22.post1, 0.7.1]
+        test-suite: [smoke-test, smoke-test-annotations]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        # 1. only install the dev requirements on top of what is in the cellxgene pip package
+        sudo apt-get update && sudo apt-get install -y libhdf5-serial-dev
+        sed -i 's/-r requirements.txt//' server/requirements-dev.txt
+        pip install -r server/requirements-dev.txt
+        # 2. install cellxgene
+        make pydist install-dist
+        # 3. install anndata
+        pip install anndata==${{ matrix.anndata-version }}
+        # workaround for anndata 0.6.22.post1 bug
+        [[ "0.6.22.post1" = "${{ matrix.anndata-version }}" ]] && pip install h5py==2.9.0 || true
+    - name: Tests
+      run: make unit-test ${{ matrix.test-suite }}
+
+  cellxgene-release-with-anndata-master:
+    name: cellxgene release with anndata master
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        test-suite: [smoke-test, smoke-test-annotations]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          path: cellxgene
+      - name: Checkout tools repo
+        uses: actions/checkout@v2
+        with:
+          repository: theislab/anndata
+          path: anndata
+      - name: Install dependencies
+        run: |
+          cd cellxgene
+          # 1. only install the dev requirements on top of what is in the cellxgene pip package
+          make dev-env-client
+          sed -i 's/-r requirements.txt//' server/requirements-dev.txt
+          pip install -r server/requirements-dev.txt
+          # 2. install cellxgene
+          pip install --upgrade cellxgene
+          # 3. install anndata
+          cd ../anndata && pip install -e .
+      - name: Tests
+        run: cd cellxgene && make unit-test ${{ matrix.test-suite }}
+
+  cellxgene-main-with-anndata-master:
+    name: cellxgene main with anndata master
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        test-suite: [smoke-test, smoke-test-annotations]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          path: cellxgene
+      - name: Checkout tools repo
+        uses: actions/checkout@v2
+        with:
+          repository: theislab/anndata
+          path: anndata
+      - name: Install dependencies
+        run: |
+          cd cellxgene
+          sed -i -E 's/^anndata[>=]=[0-9]+.[0-9]+.[0-9]+$/anndata/g' server/requirements.txt
+          make pydist install-dist dev-env
+          cd ../anndata
+          pip install -e .
+      - name: Tests
+        run: cd cellxgene && make unit-test ${{ matrix.test-suite }}

.github/workflows/deploy.yml

@@ -0,0 +1,13 @@
+name: Deploy via single cell infra repo
+
+on:
+  push:
+    branches: main
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: repository dispatch
+        run: |
+          curl -XPOST -u czi-sci-single-cell-eng:${{secrets.SCI_GITHUB_TOKEN}} -H "Accept: application/vnd.github.everest-preview+json"  -H "Content-Type: application/json" https://api.github.com/repos/chanzuckerberg/single-cell-infra/dispatches --data '{"event_type": "cellxgene-hook"}'

.github/workflows/push_tests.yml

@@ -0,0 +1,130 @@
+name: Push Tests
+
+on:
+  push:
+    branches: main
+  pull_request:
+    branches: "*"
+
+env:
+  JEST_ENV: prod
+  CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: |
+          git fetch --depth=1 origin +${{github.base_ref}}
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Node cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: |
+          pip install flake8
+          cd client
+          npm install
+      - name: Lint with flake8
+        run: |
+          make lint-server
+      - name: Lint src with eslint
+        working-directory: ./client
+        run: |
+          make lint
+
+  unit-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Python cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Node cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: make pydist install-dist dev-env-server
+      - name: Unit tests
+        run: |
+          make unit-test
+          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF backend,python,unitTest
+          cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
+
+  smoke-tests:
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Python cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Node cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: make pydist install-dist
+      - name: Smoke tests (without annotations feature)
+        run: |
+          cd client && make smoke-test
+          ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTest
+
+  smoke-tests-annotations:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Python cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Node cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: make pydist install-dist
+      - name: Smoke tests (with annotations feature)
+        run: |
+          cd client && make smoke-test-annotations
+          ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTestAnnotations

.gitignore

@@ -1,16 +1,56 @@
 # dependencies
-node_modules
+node_modules/
 
 # coverage
 coverage
+coverage.xml
+.coverage
 .nyc_output
 
+.pytest_cache/
+
 # production
-build
+build/
+dist/
+*.egg-info
+
+# Environments
+venv/
+cellxgene/
+
+# client build
+server/common/web/static/*
+server/common/web/templates/
+server/common/web/csp-hashes.json
+
+# eb build
+artifact.dir
+artifact.zip
+customize
+
+# Jupyter Notebook
+.ipynb_checkpoints
+*.ipynb
+
+# cefpython
+error.log
 
 # misc
 .DS_Store
 npm-debug.log
 .vscode
-
+*.idea*
+__pycache__
+*.DS_Store*
 data
+tags
+myconfig.yaml
+
+# Jekyll
+docs/_site/
+docs/Gemfile.lock
+
+client/.eslintcache
+
+# E2E Testing
+ignoreE2E*

CONTRIBUTING.md

@@ -0,0 +1,3 @@
+We warmly welcome contributions from the community!
+
+Whether you want to contribute ideas, requests, documentation, or code, you can get started by visiting our [contribution guide](https://chanzuckerberg.github.io/cellxgene/posts/contribute).

Dockerfile

@@ -0,0 +1,10 @@
+FROM ubuntu:bionic
+
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+
+RUN apt-get update && \
+    apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests && \
+    pip3 install cellxgene
+
+ENTRYPOINT ["cellxgene"]

MANIFEST.in

@@ -0,0 +1,5 @@
+recursive-include server/common/web/templates *
+recursive-include server/common/web/static *
+
+include server/requirements.txt
+include server/requirements-prepare.txt

Makefile

@@ -0,0 +1,185 @@
+include common.mk
+
+BUILDDIR := build
+CLIENTBUILD := $(BUILDDIR)/client
+SERVERBUILD := $(BUILDDIR)/server
+CLEANFILES :=  $(BUILDDIR)/ client/build build dist cellxgene.egg-info
+
+PART ?= patch
+
+# CLEANING
+.PHONY: clean
+clean: clean-lite clean-server clean-client
+
+# cleaning the client's node_modules is the longest one, so we avoid that if possible
+.PHONY: clean-lite
+clean-lite:
+	rm -rf $(CLEANFILES)
+
+clean-%:
+	cd $(*) && $(MAKE) clean
+
+
+# BUILDING PACKAGE
+
+.PHONY: build-client
+build-client:
+	cd client && $(MAKE) ci build
+
+.PHONY: build
+build: clean build-client
+	git ls-files server/ | grep -v 'server/test/' | cpio -pdm $(BUILDDIR)
+	cp -r client/build/  $(CLIENTBUILD)
+	$(call copy_client_assets,$(CLIENTBUILD),$(SERVERBUILD))
+	cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR)
+
+# If you are actively developing in the server folder use this, dirties the source tree
+.PHONY: build-for-server-dev
+build-for-server-dev: clean-server build-client
+	$(call copy_client_assets,client/build,server)
+
+.PHONY: copy-client-assets
+copy-client-assets:
+	$(call copy_client_assets,client/build,server)
+
+# TESTING
+.PHONY: test
+test: unit-test smoke-test
+
+.PHONY: unit-test
+unit-test: unit-test-server unit-test-client
+
+unit-test-%:
+	cd $(*) && $(MAKE) unit-test
+
+.PHONY: smoke-test
+smoke-test:
+	cd client && $(MAKE) smoke-test
+
+.PHONY: smoke-test-annotations
+smoke-test-annotations:
+	cd client && $(MAKE) smoke-test-annotations
+
+.PHONY: test-db
+test-db:
+	cd server && $(MAKE) test-db
+
+
+# FORMATTING CODE
+
+.PHOHY: fmt
+fmt: fmt-client fmt-py
+
+.PHONY: fmt-client
+fmt-client:
+	cd client && $(MAKE) fmt
+
+.PHONY: fmt
+fmt-py:
+	black .
+
+.PHONY: lint
+lint: lint-server lint-client
+
+.PHONY: lint-server
+lint-server:
+	flake8 server
+
+.PHONY: lint-client
+lint-client:
+	cd client && $(MAKE) lint
+
+# CREATING DISTRIBUTION RELEASE
+
+.PHONY: pydist
+pydist: build
+	cd $(BUILDDIR); python setup.py sdist -d ../dist
+	@echo "done"
+
+
+# RELEASE HELPERS
+
+# create new version to commit to main
+.PHONY: release-stage-1
+release-stage-1: dev-env bump clean-lite gen-package-lock
+	@echo "Version bumped part:$(PART) and client built. Ready to commit and push"
+
+# build dist and release to dev pypi
+.PHONY: release-stage-2
+release-stage-2: dev-env pydist twine
+	@echo "Dist built and uploaded to test.pypi.org"
+	@echo "Test the install:"
+	@echo "    make install-release-test"
+	@echo "Then upload to Pypi prod:"
+	@echo "    make twine-prod"
+
+.PHONY: release-stage-final
+release-stage-final: twine-prod
+	@echo "Release uploaded to pypi.org"
+
+# DANGER: releases directly to prod
+# use this if you accidently burned a test release version number,
+.PHONY: release-directly-to-prod
+release-directly-to-prod: dev-env pydist twine-prod
+	@echo "Dist built and uploaded to pypi.org"
+	@echo "Test the install:"
+	@echo "    make install-release"
+
+.PHONY: dev-env
+dev-env: dev-env-client dev-env-server
+
+.PHONY: dev-env-client
+dev-env-client:
+	cd client && $(MAKE) ci
+
+.PHONY: dev-env-server
+dev-env-server:
+	pip install -r server/requirements-dev.txt
+
+# give PART=[major, minor, part] as param to make bump
+.PHONY: bump
+bump:
+	bumpversion --config-file .bumpversion.cfg $(PART)
+
+.PHONY: twine
+twine:
+	twine upload --repository-url https://test.pypi.org/legacy/ dist/*
+
+.PHONY: twine-prod
+twine-prod:
+	twine upload dist/*
+
+# quicker than re-building client
+.PHONY: gen-package-lock
+gen-package-lock:
+	cd client && $(MAKE) install
+
+
+# INSTALL
+
+# install from source tree for development
+.PHONY: install-dev
+install-dev: uninstall
+	pip install -e .
+
+# install from test.pypi to test your release
+.PHONY: install-release-test
+install-release-test: uninstall
+	pip install --no-cache-dir --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple cellxgene
+	@echo "Installed cellxgene from test.pypi.org, now run and smoke test"
+
+# install from pypi to test your release
+.PHONY: install-release
+install-release: uninstall
+	pip install --no-cache-dir cellxgene
+	@echo "Installed cellxgene from pypi.org"
+
+# install from dist
+.PHONY: install-dist
+install-dist: uninstall
+	pip install dist/cellxgene*.tar.gz
+
+.PHONY: uninstall
+uninstall:
+	pip uninstall -y cellxgene || :
+

Procfile

@@ -0,0 +1 @@
+web: gunicorn --chdir server/eb app:application --log-file -

README.md

@@ -1,9 +1,106 @@
-# cellxgene
+<img src="./docs/cellxgene-logo.png" width="300">
 
-A React + Redux web application for exploring large scale single cell RNA sequence data.
+_an interactive explorer for single-cell transcriptomics data_
 
-##### Quickstart:
+[![DOI](https://zenodo.org/badge/105615409.svg)](https://zenodo.org/badge/latestdoi/105615409) [![PyPI](https://img.shields.io/pypi/v/cellxgene)](https://pypi.org/project/cellxgene/) [![PyPI - Downloads](https://img.shields.io/pypi/dm/cellxgene)](https://pypistats.org/packages/cellxgene) [![GitHub last commit](https://img.shields.io/github/last-commit/chanzuckerberg/cellxgene)](https://github.com/chanzuckerberg/cellxgene/pulse)
+[![Push Tests](https://github.com/chanzuckerberg/cellxgene/workflows/Push%20Tests/badge.svg)](https://github.com/chanzuckerberg/cellxgene/actions?query=workflow%3A%22Push+Tests%22)
+[![Compatibility Tests](https://github.com/chanzuckerberg/cellxgene/workflows/Compatibility%20Tests/badge.svg)](https://github.com/chanzuckerberg/cellxgene/actions?query=workflow%3A%22Compatibility+Tests%22)
+![Code Coverage](https://codecov.io/gh/chanzuckerberg/cellxgene/branch/main/graph/badge.svg)
 
-* `npm install`
+cellxgene (pronounced "cell-by-gene") is an interactive data explorer for single-cell transcriptomics datasets, such as those coming from the [Human Cell Atlas](https://humancellatlas.org). Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data.
-* `npm start`
+
-* `localhost:3000`
+Whether you need to visualize one thousand cells or one million, cellxgene helps you gain insight into your single-cell data.
+
+<img src="https://github.com/chanzuckerberg/cellxgene/raw/main/docs/images/crossfilter.gif" width="350" height="200" hspace="30"><img src="https://github.com/chanzuckerberg/cellxgene/raw/main/docs/images/category-breakdown.gif" width="350" height="200" hspace="30">
+
+# Getting started
+
+### The comprehensive guide to cellxgene
+
+[The cellxgene documentation is your one-stop-shop for information about cellxgene](https://chanzuckerberg.github.io/cellxgene/)! You may be particularly interested in:
+
+- Seeing [what cellxgene can do](https://chanzuckerberg.github.io/cellxgene/posts/gallery)
+- Learning more about cellxgene [installation](https://chanzuckerberg.github.io/cellxgene/posts/install) and [usage](https://chanzuckerberg.github.io/cellxgene/posts/launch)
+- [Preparing your own data](https://chanzuckerberg.github.io/cellxgene/posts/prepare) for use in cellxgene
+- Checking out [our roadmap](https://chanzuckerberg.github.io/cellxgene/posts/roadmap) for future development
+- [Contributing](https://chanzuckerberg.github.io/cellxgene/posts/contribute) to cellxgene
+
+### Quick start
+
+To install cellxgene you need Python 3.6+. We recommend [installing cellxgene into a conda or virtual environment.](https://chanzuckerberg.github.io/cellxgene/posts/install)
+
+Install the package.
+
+```bash
+pip install cellxgene
+```
+
+Launch cellxgene with an example [anndata](https://anndata.readthedocs.io/en/latest/) file
+
+```bash
+cellxgene launch https://cellxgene-example-data.czi.technology/pbmc3k.h5ad
+```
+
+To explore more datasets already formatted for cellxgene, check out the [Demo data](https://chanzuckerberg.github.io/cellxgene/posts/demo-data) or
+see [Preparing your data](https://chanzuckerberg.github.io/cellxgene/posts/prepare) to learn more about formatting your own
+data for cellxgene.
+
+### Supported browsers
+
+cellxgene currently supports the following browsers:
+
+- Google Chrome 61+
+- Edge 15+
+- Firefox 60+
+- Safari 10.1+
+
+Please [file an issue](https://github.com/chanzuckerberg/cellxgene/issues/new/choose) if you would like us to add support for an unsupported browser.
+
+### Finding help
+
+We'd love to hear from you!
+For questions, suggestions, or accolades, [join the `#cellxgene-users` channel on the CZI Science Slack](https://join-cellxgene-users.herokuapp.com/) and say "hi!".
+
+For any errors, [report bugs on Github](https://github.com/chanzuckerberg/cellxgene/issues).
+
+# Developing with cellxgene
+
+### Contributing
+
+We warmly welcome contributions from the community! Please see our [contributing guide](https://chanzuckerberg.github.io/cellxgene/posts/contribute) and don't hesitate to open an issue or send a pull request to improve cellxgene.
+
+This project adheres to the Contributor Covenant [code of conduct](https://github.com/chanzuckerberg/.github/blob/master/CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to opensource@chanzuckerberg.com.
+
+### Reuse
+
+This project was started with the sole goal of empowering the scientific community to explore and understand their data. As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from this project, and reach out to us with ideas or questions. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
+
+### Security
+
+If you believe you have found a security issue, we would appreciate notification. Please send email to <security@chanzuckerberg.com>.
+
+# About
+
+### Core team
+
+The current core team:
+
+- Colin Megill, frontend & product design
+- Bruce Martin, software engineer
+- Sidney Bell, computational biologist
+- Lia Prins, designer
+- Severiano Badajoz, software engineer
+
+We would also like to gratefully acknowledge contributions from past core team members:
+
+- Charlotte Weaver, software engineer
+
+### Inspiration
+
+We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browswer](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [Gene Pattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data.
+
+We were inspired by Mike Bostock and the [crossfilter](https://github.com/crossfilter) team for the design of our filtering implementation.
+
+We have been working closely with the [scanpy](https://github.com/theislab/scanpy) team to integrate with their awesome analysis tools. Special thanks to Alex Wolf, Fabian Theis, and the rest of the team for their help during development and for providing an example dataset.
+
+We are eager to explore integrations with other computational backends such as [Seurat](https://github.com/satijalab/seurat) or [Bioconductor](https://github.com/Bioconductor)

app.json

@@ -0,0 +1,29 @@
+{
+  "name": "cellxgene",
+  "description": "An interactive explorer for single-cell transcriptomics data",
+  "repository": "https://github.com/chanzuckerberg/cellxgene",
+  "logo": "https://cellxgene-example-data.czi.technology/favicon.png",
+  "keywords": [
+    "scientific",
+    "visualization",
+    "scrna-seq",
+    "transcriptomics",
+    "dataviz"
+  ],
+  "buildpacks": [
+    {
+      "url": "heroku/nodejs"
+    },
+    {
+      "url": "heroku/python"
+    }
+  ],
+  "stack": "heroku-18",
+  "env": {
+    "DATASET": {
+      "description": "Link to dataset",
+      "value": "https://cellxgene-example-data.czi.technology/pbmc3k.h5ad",
+      "required": "true"
+    }
+  }
+}

client/Makefile

@@ -0,0 +1,55 @@
+include ../common.mk
+
+ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../server/test/fixtures/pbmc3k-annotations.csv)
+ANNOTATIONS_FILENAME := $(shell basename $(ANNOTATIONS))
+
+# Packaging
+.PHONY: clean
+clean:
+	rm -rf node_modules
+	rm -f __tests__/screenshots/*.png
+
+.PHONY: ci
+ci:
+	npm ci
+
+.PHONY: install
+install:
+	npm install
+
+.PHONY: build
+WEBPACK_CONFIG ?= configuration/webpack/webpack.config.prod.js
+build:
+	npm run build $(WEBPACK_CONFIG)
+
+# Development convenience methods
+.PHONY: start-frontend
+start-frontend:
+	node server/development.js
+
+# start an instance of cellxgene and run the end-to-end tests
+.PHONY: smoke-test
+smoke-test:
+	start_server_and_test \
+		'CXG_OPTIONS="--disable-annotations" $(MAKE) start-server' \
+		$(CXG_SERVER_PORT) \
+		'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" npm run e2e -- --verbose false'
+
+# start an instance of cellxgene and run the end-to-end annotations tests
+.PHONY: smoke-test-annotations
+smoke-test-annotations:
+	$(eval TMP_DIR := $(shell  mktemp -d /tmp/cellxgene_XXXXXX))
+	cp $(ANNOTATIONS) $(TMP_DIR)/ && \
+	start_server_and_test \
+		'CXG_OPTIONS="--annotations-file $(TMP_DIR)/$(ANNOTATIONS_FILENAME)" $(MAKE) start-server' \
+		$(CXG_SERVER_PORT) \
+		'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" npm run e2e-annotations -- --verbose false'
+	rm -rf $(TMP_DIR)
+
+.PHONY: unit-test
+unit-test:
+	node node_modules/jest/bin/jest.js --testPathIgnorePatterns e2e
+
+# pass remaining commands through to npm run
+%:
+	npm run $(*)

client/__tests__/e2e/__snapshots__/e2e.test.js.snap

@@ -0,0 +1,5 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`did launch page launched 1`] = `"<span style=\\"width: 185px; display: flex; overflow: hidden; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">pbm</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">c3k</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">c3k</span></span></span>"`;
+
+exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testclass=\\"category-expand\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><span data-testid=\\"louvain:category-label\\" aria-label=\\"louvain\\" class=\\"\\" tabindex=\\"0\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">vain</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">vain</span></span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><desc>tint</desc><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"></div><div></div>"`;

client/__tests__/e2e/__snapshots__/e2eAnnotations.test.js.snap

@@ -0,0 +1,23 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`annotations stacked bar graph renders 1`] = `
+Array [
+  "<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-TEST-LABEL\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value\\" aria-label=\\"TEST-LABEL\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 63px; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">TEST-</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">LABEL</span><span style=\\"position: absolute; right: 0px; color: black;\\">LABEL</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-TEST-LABEL\\" style=\\"color: black;\\">0</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:TEST-LABEL:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
+  "<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-unassigned\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value\\" aria-label=\\"unassigned\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">unass</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">igned</span><span style=\\"position: absolute; right: 0px; color: rgb(171, 171, 171);\\">igned</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"><canvas class=\\"bp3-popover-targer\\" width=\\"100\\" height=\\"11\\" style=\\"margin-right: 5px; width: 100px; height: 11px;\\"></canvas></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-unassigned\\" style=\\"color: rgb(171, 171, 171); font-style: italic;\\">2133</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:unassigned:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
+]
+`;
+
+exports[`annotations stacked bar graph renders 2`] = `
+Array [
+  "<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-TEST-LABEL\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value\\" aria-label=\\"TEST-LABEL\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 63px; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">TEST-</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">LABEL</span><span style=\\"position: absolute; right: 0px; color: black;\\">LABEL</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-TEST-LABEL\\" style=\\"color: black;\\">0</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:TEST-LABEL:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
+  "<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-unassigned\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value\\" aria-label=\\"unassigned\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">unass</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">igned</span><span style=\\"position: absolute; right: 0px; color: rgb(171, 171, 171);\\">igned</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"><canvas class=\\"bp3-popover-targer\\" width=\\"100\\" height=\\"11\\" style=\\"margin-right: 5px; width: 100px; height: 11px;\\"></canvas></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-unassigned\\" style=\\"color: rgb(171, 171, 171); font-style: italic;\\">2638</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:unassigned:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
+]
+`;
+
+exports[`annotations truncate midpoint whitespace 1`] = `"<span data-testid=\\"categorical-value-TEST-CATEGORY-123 456\\" data-testclass=\\"categorical-value\\" aria-label=\\"123 456\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 187px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 187px; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">123</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">&nbsp;456</span><span style=\\"position: absolute; right: 0px; color: black;\\">&nbsp;456</span></span></span></span>"`;
+
+exports[`annotations truncate midpoint whitespace 2`] = `"<span data-testid=\\"categorical-value-TEST-CATEGORY-123 456\\" data-testclass=\\"categorical-value\\" aria-label=\\"123 456\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 187px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 187px; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">123</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">&nbsp;456</span><span style=\\"position: absolute; right: 0px; color: black;\\">&nbsp;456</span></span></span></span>"`;
+
+exports[`annotations truncate single character 1`] = `"<span data-testid=\\"categorical-value-TEST-CATEGORY-T\\" data-testclass=\\"categorical-value\\" aria-label=\\"T\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 187px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 187px; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">T</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\"></span><span style=\\"position: absolute; right: 0px; color: black;\\"></span></span></span></span>"`;
+
+exports[`annotations truncate single character 2`] = `"<span data-testid=\\"categorical-value-TEST-CATEGORY-T\\" data-testclass=\\"categorical-value\\" aria-label=\\"T\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 187px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 187px; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">T</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\"></span><span style=\\"position: absolute; right: 0px; color: black;\\"></span></span></span></span>"`;

client/__tests__/e2e/cellxgeneActions.js

@@ -0,0 +1,315 @@
+/* eslint-disable no-await-in-loop -- await in loop is needed to emulate sequential user actions  */
+import { strict as assert } from "assert";
+import {
+  clearInputAndTypeInto,
+  clickOn,
+  getAllByClass,
+  getOneElementInnerText,
+  typeInto,
+  waitByID,
+  waitByClass,
+  waitForAllByIds,
+  clickOnUntil,
+  getTestClass,
+  getTestId,
+  isElementPresent,
+} from "./puppeteerUtils";
+
+export async function drag(testId, start, end, lasso = false) {
+  const layout = await waitByID(testId);
+  const elBox = await layout.boxModel();
+  const x1 = elBox.content[0].x + start.x;
+  const x2 = elBox.content[0].x + end.x;
+  const y1 = elBox.content[0].y + start.y;
+  const y2 = elBox.content[0].y + end.y;
+  await page.mouse.move(x1, y1);
+  await page.mouse.down();
+  if (lasso) {
+    await page.mouse.move(x2, y1);
+    await page.mouse.move(x2, y2);
+    await page.mouse.move(x1, y2);
+    await page.mouse.move(x1, y1);
+  } else {
+    await page.mouse.move(x2, y2);
+  }
+  await page.mouse.up();
+}
+
+export async function clickOnCoordinate(testId, coord) {
+  const layout = await expect(page).toMatchElement(getTestId(testId));
+  const elBox = await layout.boxModel();
+
+  if (!elBox) {
+    throw Error("Layout's boxModel is not available!");
+  }
+
+  const x = elBox.content[0].x + coord.x;
+  const y = elBox.content[0].y + coord.y;
+  await page.mouse.click(x, y);
+}
+
+export async function getAllHistograms(testclass, testIds) {
+  const histTestIds = testIds.map((tid) => `histogram-${tid}`);
+
+  // these load asynchronously, so we need to wait for each histogram individually,
+  // and they may be quite slow in some cases.
+  await waitForAllByIds(histTestIds, { timeout: 4 * 60 * 1000 });
+
+  const allHistograms = await getAllByClass(testclass);
+
+  const testIDs = await Promise.all(
+    allHistograms.map((hist) => {
+      return page.evaluate((elem) => {
+        return elem.dataset.testid;
+      }, hist);
+    })
+  );
+
+  return testIDs.map((id) => id.replace(/^histogram-/, ""));
+}
+
+export async function getAllCategoriesAndCounts(category) {
+  // these load asynchronously, so we have to wait for the specific category.
+  await waitByID(`category-${category}`);
+
+  return page.$$eval(
+    `[data-testid="category-${category}"] [data-testclass='categorical-row']`,
+    (rows) =>
+      Object.fromEntries(
+        rows.map((row) => {
+          const cat = row
+            .querySelector("[data-testclass='categorical-value']")
+            .getAttribute("aria-label");
+
+          const count = row.querySelector(
+            "[data-testclass='categorical-value-count']"
+          ).innerText;
+
+          return [cat, count];
+        })
+      )
+  );
+}
+
+export async function getCellSetCount(num) {
+  await clickOn(`cellset-button-${num}`);
+  return getOneElementInnerText(`[data-testid='cellset-count-${num}']`);
+}
+
+export async function resetCategory(category) {
+  const checkboxId = `${category}:category-select`;
+  await waitByID(checkboxId);
+  const checkedPseudoclass = await page.$eval(
+    `[data-testid='${checkboxId}']`,
+    (el) => el.matches(":checked")
+  );
+  if (!checkedPseudoclass) await clickOn(checkboxId);
+
+  const categoryRow = await waitByID(`${category}:category-expand`);
+
+  const isExpanded = await categoryRow.$(
+    "[data-testclass='category-expand-is-expanded']"
+  );
+
+  if (isExpanded) await clickOn(`${category}:category-expand`);
+}
+
+export async function calcCoordinate(testId, xAsPercent, yAsPercent) {
+  const el = await waitByID(testId);
+  const size = await el.boxModel();
+  return {
+    x: Math.floor(size.width * xAsPercent),
+    y: Math.floor(size.height * yAsPercent),
+  };
+}
+
+export async function calcDragCoordinates(testId, coordinateAsPercent) {
+  return {
+    start: await calcCoordinate(
+      testId,
+      coordinateAsPercent.x1,
+      coordinateAsPercent.y1
+    ),
+    end: await calcCoordinate(
+      testId,
+      coordinateAsPercent.x2,
+      coordinateAsPercent.y2
+    ),
+  };
+}
+
+export async function selectCategory(category, values, reset = true) {
+  if (reset) await resetCategory(category);
+
+  await clickOn(`${category}:category-expand`);
+  await clickOn(`${category}:category-select`);
+
+  for (const value of values) {
+    await clickOn(`categorical-value-select-${category}-${value}`);
+  }
+}
+
+export async function expandCategory(category) {
+  const expand = await waitByID(`${category}:category-expand`);
+  const notExpanded = await expand.$(
+    "[data-testclass='category-expand-is-not-expanded']"
+  );
+  if (notExpanded) await clickOn(`${category}:category-expand`);
+}
+
+export async function clip(min = 0, max = 100) {
+  await clickOn("visualization-settings");
+  await clearInputAndTypeInto("clip-min-input", min);
+  await clearInputAndTypeInto("clip-max-input", max);
+  await clickOn("clip-commit");
+}
+
+export async function createCategory(categoryName) {
+  await clickOnUntil("open-annotation-dialog", async () => {
+    await expect(page).toMatchElement(getTestId("new-category-name"));
+  });
+
+  await typeInto("new-category-name", categoryName);
+  await clickOn("submit-category");
+}
+
+export async function duplicateCategory(categoryName) {
+  await clickOn("open-annotation-dialog");
+
+  await typeInto("new-category-name", categoryName);
+
+  const dropdownOptionClass = "duplicate-category-dropdown-option";
+
+  await clickOnUntil("duplicate-category-dropdown", async () => {
+    await expect(page).toMatchElement(getTestClass(dropdownOptionClass));
+  });
+
+  const option = await expect(page).toMatchElement(
+    getTestClass(dropdownOptionClass)
+  );
+
+  await option.click();
+
+  await clickOnUntil("submit-category", async () => {
+    await expect(page).toMatchElement(
+      getTestId(`${categoryName}:category-expand`)
+    );
+  });
+
+  await waitByClass("autosave-complete");
+}
+
+export async function renameCategory(oldCategoryName, newCategoryName) {
+  await clickOn(`${oldCategoryName}:see-actions`);
+  await clickOn(`${oldCategoryName}:edit-category-mode`);
+  await clearInputAndTypeInto(
+    `${oldCategoryName}:edit-category-name-text`,
+    newCategoryName
+  );
+  await clickOn(`${oldCategoryName}:submit-category-edit`);
+}
+
+export async function deleteCategory(categoryName) {
+  const targetId = `${categoryName}:delete-category`;
+
+  await clickOnUntil(`${categoryName}:see-actions`, async () => {
+    await expect(page).toMatchElement(getTestId(targetId));
+  });
+
+  await clickOn(targetId);
+
+  await assertCategoryDoesNotExist();
+}
+
+export async function createLabel(categoryName, labelName) {
+  /**
+   * (thuang): This explicit wait is needed, since currently showing
+   * the modal again quickly after the previous action dismissing the
+   * modal will persist the input value from the previous action.
+   *
+   * To reproduce:
+   * 1. Click on the plus sign to show the modal to add a new label to the category
+   * 2. Type `123` in the input box
+   * 3. Hover over your mouse over the plus sign and double click to quickly dismiss and
+   * invoke the modal again
+   * 4. You will see `123` is persisted in the input box
+   * 5. Expected behavior is to get an empty input box
+   */
+  await page.waitFor(500);
+
+  await clickOn(`${categoryName}:see-actions`);
+
+  await clickOn(`${categoryName}:add-new-label-to-category`);
+
+  await typeInto(`${categoryName}:new-label-name`, labelName);
+
+  await clickOn(`${categoryName}:submit-label`);
+}
+
+export async function deleteLabel(categoryName, labelName) {
+  await expandCategory(categoryName);
+  await clickOn(`${categoryName}:${labelName}:see-actions`);
+  await clickOn(`${categoryName}:${labelName}:delete-label`);
+}
+
+export async function renameLabel(categoryName, oldLabelName, newLabelName) {
+  await expandCategory(categoryName);
+  await clickOn(`${categoryName}:${oldLabelName}:see-actions`);
+  await clickOn(`${categoryName}:${oldLabelName}:edit-label`);
+  await clearInputAndTypeInto(
+    `${categoryName}:${oldLabelName}:edit-label-name`,
+    newLabelName
+  );
+  await clickOn(`${categoryName}:${oldLabelName}:submit-label-edit`);
+}
+
+export async function addGeneToSearch(geneName) {
+  await typeInto("gene-search", geneName);
+  await page.keyboard.press("Enter");
+  await page.waitForSelector(`[data-testid='histogram-${geneName}']`);
+}
+
+export async function subset(coordinatesAsPercent) {
+  // In order to deselect the selection after the subset, make sure we have some clear part
+  // of the scatterplot we can click on
+  assert(coordinatesAsPercent.x2 < 0.99 || coordinatesAsPercent.y2 < 0.99);
+  const lassoSelection = await calcDragCoordinates(
+    "layout-graph",
+    coordinatesAsPercent
+  );
+  await drag("layout-graph", lassoSelection.start, lassoSelection.end, true);
+  await clickOn("subset-button");
+  const clearCoordinate = await calcCoordinate("layout-graph", 0.5, 0.99);
+  await clickOnCoordinate("layout-graph", clearCoordinate);
+}
+
+export async function setSellSet(cellSet, cellSetNum) {
+  const selections = cellSet.filter((sel) => sel.kind === "categorical");
+
+  for (const selection of selections) {
+    await selectCategory(selection.metadata, selection.values, true);
+  }
+
+  await getCellSetCount(cellSetNum);
+}
+
+export async function runDiffExp(cellSet1, cellSet2) {
+  await setSellSet(cellSet1, 1);
+  await setSellSet(cellSet2, 2);
+  await clickOn("diffexp-button");
+}
+
+export async function bulkAddGenes(geneNames) {
+  await clickOn("section-bulk-add");
+  await typeInto("input-bulk-add", geneNames.join(","));
+  await page.keyboard.press("Enter");
+}
+
+export async function assertCategoryDoesNotExist(categoryName) {
+  const result = await isElementPresent(
+    getTestId(`${categoryName}:category-label`)
+  );
+
+  await expect(result).toBe(false);
+}
+/* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/config.js

@@ -0,0 +1,8 @@
+import * as ENV_DEFAULT from "../../../environment.default.json";
+
+export const jestEnv = process.env.JEST_ENV || ENV_DEFAULT.JEST_ENV;
+export const appUrlBase =
+  process.env.CXG_URL_BASE || `http://localhost:${ENV_DEFAULT.CXG_CLIENT_PORT}`;
+export const DATASET = "pbmc3k";
+export const isDev = jestEnv === ENV_DEFAULT.DEV;
+export const isDebug = jestEnv === ENV_DEFAULT.DEBUG;

client/__tests__/e2e/data.js

@@ -0,0 +1,147 @@
+export const datasets = {
+  pbmc3k: {
+    title: "pbmc3k",
+    dataframe: {
+      nObs: "2638",
+      nVar: "1838",
+      type: "float32",
+    },
+    categorical: {
+      louvain: {
+        "B cells": "342",
+        "CD14+ Monocytes": "480",
+        "CD4 T cells": "1144",
+        "CD8 T cells": "316",
+        "Dendritic cells": "37",
+        "FCGR3A+ Monocytes": "150",
+        Megakaryocytes: "15",
+        "NK cells": "154",
+      },
+    },
+    continuous: {
+      n_genes: "int32",
+      percent_mito: "float32",
+      n_counts: "float32",
+    },
+    cellsets: {
+      lasso: [
+        {
+          "coordinates-as-percent": { x1: 0.1, y1: 0.25, x2: 0.7, y2: 0.75 },
+          count: "1131",
+        },
+      ],
+      categorical: [
+        {
+          metadata: "louvain",
+          values: ["B cells", "Megakaryocytes"],
+          count: "357",
+        },
+      ],
+      continuous: [
+        {
+          metadata: "n_genes",
+          "coordinates-as-percent": { x1: 0.25, y1: 0.5, x2: 0.55, y2: 0.5 },
+          count: "1537",
+        },
+      ],
+    },
+
+    diffexp: {
+      cellset1: [
+        { kind: "categorical", metadata: "louvain", values: ["B cells"] },
+      ],
+      cellset2: [
+        {
+          kind: "categorical",
+          metadata: "louvain",
+          values: ["CD4 T cells", "NK cells"],
+        },
+      ],
+      "gene-results": [
+        "HLA-DRB1",
+        "HLA-DPB1",
+        "CD79A",
+        "HLA-DPA1",
+        "HLA-DQA1",
+        "CD79B",
+        "HLA-DQB1",
+        "MS4A1",
+        "IL32",
+        "CD37",
+      ],
+    },
+
+    genes: {
+      bulkadd: ["S100A8", "FCGR3A", "LGALS2", "GSTP1"],
+      search: "ACD",
+    },
+    subset: {
+      cellset1: [
+        {
+          kind: "categorical",
+          metadata: "louvain",
+          values: ["B cells", "Megakaryocytes"],
+        },
+      ],
+      count: "357",
+      categorical: {
+        louvain: {
+          "B cells": "342",
+          "CD14+ Monocytes": "0",
+          "CD4 T cells": "0",
+          "CD8 T cells": "0",
+          "Dendritic cells": "0",
+          "FCGR3A+ Monocytes": "0",
+          Megakaryocytes: "15",
+          "NK cells": "0",
+        },
+      },
+      lasso: {
+        "coordinates-as-percent": { x1: 0.25, y1: 0.05, x2: 0.75, y2: 0.55 },
+        count: "331",
+      },
+    },
+    scatter: {
+      genes: { x: "S100A8", y: "FCGR3A" },
+    },
+    pan: {
+      "coordinates-as-percent": { x1: 0.75, y1: 0.75, x2: 0.35, y2: 0.35 },
+    },
+    features: {
+      panzoom: {
+        lasso: {
+          "coordinates-as-percent": { x1: 0.3, y1: 0.3, x2: 0.5, y2: 0.5 },
+          count: "38",
+        },
+      },
+    },
+    categoryLabel: {
+      lasso: {
+        "coordinates-as-percent": { x1: 0.05, y1: 0.3, x2: 0.5, y2: 0.5 },
+      },
+      newCount: {
+        bySubsetConfig: {
+          false: "668",
+          true: "659",
+        },
+      },
+    },
+    annotationsFromFile: {
+      count: {
+        bySubsetConfig: {
+          false: "1161",
+          true: "852",
+        },
+      },
+    },
+    clip: {
+      min: "30",
+      max: "70",
+      metadata: "n_genes",
+      gene: "S100A8",
+      "coordinates-as-percent": { x1: 0.25, y1: 0.5, x2: 0.55, y2: 0.5 },
+      count: "386",
+      "gene-cell-count": "416",
+    },
+  },
+};

client/__tests__/e2e/e2e.test.js

@@ -0,0 +1,521 @@
+/**
+ * Smoke test suite that will be run in Travis CI
+ * Tests included in this file are expected to be relatively stable and test core features
+ */
+
+/* eslint-disable no-await-in-loop -- await in loop is needed to emulate sequential user actions  */
+import { appUrlBase, DATASET } from "./config";
+
+import { datasets } from "./data";
+
+import {
+  clickOn,
+  getAllByClass,
+  getElementCoordinates,
+  getOneElementInnerHTML,
+  getTestId,
+  goToPage,
+  typeInto,
+  waitByID,
+} from "./puppeteerUtils";
+
+import {
+  addGeneToSearch,
+  bulkAddGenes,
+  calcDragCoordinates,
+  clip,
+  drag,
+  getAllCategoriesAndCounts,
+  getAllHistograms,
+  getCellSetCount,
+  runDiffExp,
+  selectCategory,
+  subset,
+} from "./cellxgeneActions";
+
+const data = datasets[DATASET];
+
+describe("did launch", () => {
+  test("page launched", async () => {
+    await goToPage(appUrlBase);
+
+    const element = await getOneElementInnerHTML(getTestId("header"));
+
+    expect(element).toMatchSnapshot();
+  });
+});
+
+describe("metadata loads", () => {
+  test("categories and values from dataset appear", async () => {
+    await goToPage(appUrlBase);
+
+    for (const label of Object.keys(data.categorical)) {
+      const element = await getOneElementInnerHTML(
+        getTestId(`category-${label}`)
+      );
+
+      expect(element).toMatchSnapshot();
+
+      await clickOn(`${label}:category-expand`);
+
+      const categories = await getAllCategoriesAndCounts(label);
+
+      expect(Object.keys(categories)).toMatchObject(
+        Object.keys(data.categorical[label])
+      );
+
+      expect(Object.values(categories)).toMatchObject(
+        Object.values(data.categorical[label])
+      );
+    }
+  });
+
+  test("continuous data appears", async () => {
+    await goToPage(appUrlBase);
+
+    for (const label of Object.keys(data.continuous)) {
+      await waitByID(`histogram-${label}`);
+    }
+  });
+});
+
+describe("cell selection", () => {
+  test("selects all cells cellset 1", async () => {
+    await goToPage(appUrlBase);
+
+    const cellCount = await getCellSetCount(1);
+    expect(cellCount).toBe(data.dataframe.nObs);
+  });
+
+  test("selects all cells cellset 2", async () => {
+    await goToPage(appUrlBase);
+
+    const cellCount = await getCellSetCount(2);
+    expect(cellCount).toBe(data.dataframe.nObs);
+  });
+
+  test("selects cells via lasso", async () => {
+    await goToPage(appUrlBase);
+
+    for (const cellset of data.cellsets.lasso) {
+      const cellset1 = await calcDragCoordinates(
+        "layout-graph",
+        cellset["coordinates-as-percent"]
+      );
+
+      await drag("layout-graph", cellset1.start, cellset1.end, true);
+      const cellCount = await getCellSetCount(1);
+      expect(cellCount).toBe(cellset.count);
+    }
+  });
+
+  test("selects cells via categorical", async () => {
+    await goToPage(appUrlBase);
+
+    for (const cellset of data.cellsets.categorical) {
+      await clickOn(`${cellset.metadata}:category-expand`);
+      await clickOn(`${cellset.metadata}:category-select`);
+
+      for (const value of cellset.values) {
+        await clickOn(`categorical-value-select-${cellset.metadata}-${value}`);
+      }
+
+      const cellCount = await getCellSetCount(1);
+
+      expect(cellCount).toBe(cellset.count);
+    }
+  });
+
+  test("selects cells via continuous", async () => {
+    await goToPage(appUrlBase);
+
+    for (const cellset of data.cellsets.continuous) {
+      const histBrushableAreaId = `histogram-${cellset.metadata}-plot-brushable-area`;
+
+      const coords = await calcDragCoordinates(
+        histBrushableAreaId,
+        cellset["coordinates-as-percent"]
+      );
+
+      await drag(histBrushableAreaId, coords.start, coords.end);
+
+      const cellCount = await getCellSetCount(1);
+
+      expect(cellCount).toBe(cellset.count);
+    }
+  });
+});
+
+describe("gene entry", () => {
+  test("search for single gene", async () => {
+    await goToPage(appUrlBase);
+    await addGeneToSearch(data.genes.search);
+  });
+
+  test("bulk add genes", async () => {
+    await goToPage(appUrlBase);
+
+    const testGenes = data.genes.bulkadd;
+
+    await bulkAddGenes(testGenes);
+    const allHistograms = await getAllHistograms(
+      "histogram-user-gene",
+      testGenes
+    );
+
+    expect(allHistograms).toEqual(expect.arrayContaining(testGenes));
+    expect(allHistograms).toHaveLength(testGenes.length);
+  });
+});
+
+describe("differential expression", () => {
+  test("selects cells, saves them and performs diffexp", async () => {
+    await goToPage(appUrlBase);
+
+    await runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
+
+    const allHistograms = await getAllHistograms(
+      "histogram-diffexp",
+      data.diffexp["gene-results"]
+    );
+
+    expect(allHistograms).toEqual(
+      expect.arrayContaining(data.diffexp["gene-results"])
+    );
+
+    expect(allHistograms).toHaveLength(data.diffexp["gene-results"].length);
+  });
+});
+
+describe("subset", () => {
+  test("subset - cell count matches", async () => {
+    await goToPage(appUrlBase);
+
+    for (const select of data.subset.cellset1) {
+      if (select.kind === "categorical") {
+        await selectCategory(select.metadata, select.values, true);
+      }
+    }
+
+    await clickOn("subset-button");
+
+    for (const label of Object.keys(data.subset.categorical)) {
+      const categories = await getAllCategoriesAndCounts(label);
+
+      expect(Object.keys(categories)).toMatchObject(
+        Object.keys(data.subset.categorical[label])
+      );
+
+      expect(Object.values(categories)).toMatchObject(
+        Object.values(data.subset.categorical[label])
+      );
+    }
+  });
+
+  test("lasso after subset", async () => {
+    await goToPage(appUrlBase);
+
+    for (const select of data.subset.cellset1) {
+      if (select.kind === "categorical") {
+        await selectCategory(select.metadata, select.values, true);
+      }
+    }
+
+    await clickOn("subset-button");
+
+    const lassoSelection = await calcDragCoordinates(
+      "layout-graph",
+      data.subset.lasso["coordinates-as-percent"]
+    );
+
+    await drag("layout-graph", lassoSelection.start, lassoSelection.end, true);
+
+    const cellCount = await getCellSetCount(1);
+    expect(cellCount).toBe(data.subset.lasso.count);
+  });
+
+  test("undo selection appends the top diff exp genes to user defined genes", async () => {
+    await goToPage(appUrlBase);
+
+    const userDefinedGenes = data.genes.bulkadd;
+    const diffExpGenes = data.diffexp["gene-results"];
+
+    await bulkAddGenes(userDefinedGenes);
+    const userDefinedHistograms = await getAllHistograms(
+      "histogram-user-gene",
+      userDefinedGenes
+    );
+
+    expect(userDefinedHistograms).toEqual(
+      expect.arrayContaining(userDefinedGenes)
+    );
+
+    await subset({ x1: 0.15, y1: 0.1, x2: 0.98, y2: 0.98 });
+    await runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
+
+    const diffExpHistograms = await getAllHistograms(
+      "histogram-diffexp",
+      diffExpGenes
+    );
+
+    expect(diffExpHistograms).toEqual(expect.arrayContaining(diffExpGenes));
+
+    await clickOn("reset-subset-button");
+    const expected = [].concat(userDefinedGenes, diffExpGenes);
+    const userDefinedHistogramsAfterSubset = await getAllHistograms(
+      "histogram-user-gene",
+      expected
+    );
+
+    expect(userDefinedHistogramsAfterSubset).toEqual(
+      expect.arrayContaining(expected)
+    );
+  });
+
+  test("subset selection appends the top diff exp genes to user defined genes", async () => {
+    await goToPage(appUrlBase);
+
+    const userDefinedGenes = data.genes.bulkadd;
+    const diffExpGenes = data.diffexp["gene-results"];
+
+    await bulkAddGenes(userDefinedGenes);
+    const userDefinedHistograms = await getAllHistograms(
+      "histogram-user-gene",
+      userDefinedGenes
+    );
+
+    expect(userDefinedHistograms).toEqual(
+      expect.arrayContaining(userDefinedGenes)
+    );
+
+    await subset({ x1: 0.15, y1: 0.1, x2: 0.98, y2: 0.98 });
+    await runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
+
+    const diffExpHistograms = await getAllHistograms(
+      "histogram-diffexp",
+      diffExpGenes
+    );
+
+    expect(diffExpHistograms).toEqual(expect.arrayContaining(diffExpGenes));
+
+    await subset({ x1: 0.16, y1: 0.11, x2: 0.97, y2: 0.97 });
+
+    const expected = [].concat(userDefinedGenes, diffExpGenes);
+    const userDefinedHistogramsAfterSubset = await getAllHistograms(
+      "histogram-user-gene",
+      expected
+    );
+
+    expect(userDefinedHistogramsAfterSubset).toEqual(
+      expect.arrayContaining(expected)
+    );
+  });
+});
+
+describe("scatter plot", () => {
+  test("scatter plot appears", async () => {
+    await goToPage(appUrlBase);
+
+    await bulkAddGenes(Object.values(data.scatter.genes));
+    await clickOn(`plot-x-${data.scatter.genes.x}`);
+    await clickOn(`plot-y-${data.scatter.genes.y}`);
+    await waitByID("scatterplot");
+  });
+});
+
+describe("clipping", () => {
+  test("clip continuous", async () => {
+    await goToPage(appUrlBase);
+
+    await clip(data.clip.min, data.clip.max);
+    const histBrushableAreaId = `histogram-${data.clip.metadata}-plot-brushable-area`;
+    const coords = await calcDragCoordinates(
+      histBrushableAreaId,
+      data.clip["coordinates-as-percent"]
+    );
+    await drag(histBrushableAreaId, coords.start, coords.end);
+    const cellCount = await getCellSetCount(1);
+    expect(cellCount).toBe(data.clip.count);
+  });
+
+  test("clip gene", async () => {
+    await goToPage(appUrlBase);
+
+    await typeInto("gene-search", data.clip.gene);
+    await page.keyboard.press("Enter");
+
+    await page.waitForSelector(`[data-testid='histogram-${data.clip.gene}']`);
+
+    await clip(data.clip.min, data.clip.max);
+
+    const histBrushableAreaId = `histogram-${data.clip.gene}-plot-brushable-area`;
+
+    const coords = await calcDragCoordinates(
+      histBrushableAreaId,
+      data.clip["coordinates-as-percent"]
+    );
+
+    await drag(histBrushableAreaId, coords.start, coords.end);
+
+    const cellCount = await getCellSetCount(1);
+
+    expect(cellCount).toBe(data.clip["gene-cell-count"]);
+  });
+});
+
+// interact with UI elements just that they do not break
+describe("ui elements don't error", () => {
+  test("color by", async () => {
+    await goToPage(appUrlBase);
+
+    const allLabels = [
+      ...Object.keys(data.categorical),
+      ...Object.keys(data.continuous),
+    ];
+
+    for (const label of allLabels) {
+      await clickOn(`colorby-${label}`);
+    }
+  });
+
+  test("color by for gene", async () => {
+    await goToPage(appUrlBase);
+
+    await typeInto("gene-search", data.genes.search);
+    await page.keyboard.press("Enter");
+    await page.waitForSelector(
+      `[data-testid='histogram-${data.genes.search}']`
+    );
+    await clickOn(`colorby-${data.genes.search}`);
+  });
+
+  test("pan and zoom", async () => {
+    await goToPage(appUrlBase);
+
+    await clickOn("mode-pan-zoom");
+    const panCoords = await calcDragCoordinates(
+      "layout-graph",
+      data.pan["coordinates-as-percent"]
+    );
+
+    await drag("layout-graph", panCoords.start, panCoords.end, false);
+
+    await page.evaluate("window.scrollBy(0, 1000);");
+  });
+});
+
+describe("centroid labels", () => {
+  test("labels are created", async () => {
+    await goToPage(appUrlBase);
+
+    const labels = Object.keys(data.categorical);
+
+    await clickOn(`colorby-${labels[0]}`);
+    await clickOn("centroid-label-toggle");
+
+    // Toggle colorby for each category and check to see if labels are generated
+    for (let i = 0, { length } = labels; i < length; i += 1) {
+      const label = labels[i];
+      // first label is already enabled
+      if (i !== 0) await clickOn(`colorby-${label}`);
+      const generatedLabels = await getAllByClass("centroid-label");
+      // Number of labels generated should be equal to size of the object
+      expect(generatedLabels).toHaveLength(
+        Object.keys(data.categorical[label]).length
+      );
+    }
+  });
+});
+
+describe("graph overlay", () => {
+  test("transform centroids correctly", async () => {
+    await goToPage(appUrlBase);
+
+    const category = Object.keys(data.categorical)[0];
+
+    await clickOn(`colorby-${category}`);
+    await clickOn("centroid-label-toggle");
+    await clickOn("mode-pan-zoom");
+
+    const panCoords = await calcDragCoordinates(
+      "layout-graph",
+      data.pan["coordinates-as-percent"]
+    );
+
+    const categoryValue = Object.keys(data.categorical[category])[0];
+    const initialCoordinates = await getElementCoordinates(
+      `${categoryValue}-centroid-label`
+    );
+
+    await drag("layout-graph", panCoords.start, panCoords.end, false);
+    const terminalCoordinates = await getElementCoordinates(
+      `${categoryValue}-centroid-label`
+    );
+
+    expect(terminalCoordinates[0] - initialCoordinates[0]).toBeCloseTo(
+      panCoords.end.x - panCoords.start.x
+    );
+    expect(terminalCoordinates[1] - initialCoordinates[1]).toBeCloseTo(
+      panCoords.end.y - panCoords.start.y
+    );
+  });
+});
+
+test("pan zoom mode resets lasso selection", async () => {
+  await goToPage(appUrlBase);
+
+  const panzoomLasso = data.features.panzoom.lasso;
+
+  const lassoSelection = await calcDragCoordinates(
+    "layout-graph",
+    panzoomLasso["coordinates-as-percent"]
+  );
+
+  await drag("layout-graph", lassoSelection.start, lassoSelection.end, true);
+  await waitByID("lasso-element", { visible: true });
+
+  const initialCount = await getCellSetCount(1);
+
+  expect(initialCount).toBe(panzoomLasso.count);
+
+  await clickOn("mode-pan-zoom");
+  await clickOn("mode-lasso");
+
+  const modeSwitchCount = await getCellSetCount(1);
+
+  expect(modeSwitchCount).toBe(initialCount);
+});
+
+test("lasso moves after pan", async () => {
+  await goToPage(appUrlBase);
+
+  const panzoomLasso = data.features.panzoom.lasso;
+  const coordinatesAsPercent = panzoomLasso["coordinates-as-percent"];
+
+  const lassoSelection = await calcDragCoordinates(
+    "layout-graph",
+    coordinatesAsPercent
+  );
+
+  await drag("layout-graph", lassoSelection.start, lassoSelection.end, true);
+  await waitByID("lasso-element", { visible: true });
+
+  const initialCount = await getCellSetCount(1);
+
+  expect(initialCount).toBe(panzoomLasso.count);
+
+  await clickOn("mode-pan-zoom");
+
+  const panCoords = await calcDragCoordinates(
+    "layout-graph",
+    coordinatesAsPercent
+  );
+
+  await drag("layout-graph", panCoords.start, panCoords.end, false);
+  await clickOn("mode-lasso");
+
+  const panCount = await getCellSetCount(2);
+
+  expect(panCount).toBe(initialCount);
+});
+/* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/e2eAnnotations.test.js

@@ -0,0 +1,355 @@
+/*
+Tests included in this file are specific to annotation features
+*/
+import { appUrlBase, DATASET } from "./config";
+import { datasets } from "./data";
+
+import {
+  clickOn,
+  goToPage,
+  waitByClass,
+  waitByID,
+  getTestId,
+  getTestClass,
+  getAllByClass,
+} from "./puppeteerUtils";
+
+import {
+  assertCategoryDoesNotExist,
+  calcDragCoordinates,
+  createCategory,
+  createLabel,
+  deleteCategory,
+  deleteLabel,
+  drag,
+  expandCategory,
+  renameCategory,
+  renameLabel,
+  subset,
+  duplicateCategory,
+} from "./cellxgeneActions";
+
+const data = datasets[DATASET];
+
+const perTestCategoryName = "TEST-CATEGORY";
+const perTestLabelName = "TEST-LABEL";
+
+async function setup(config) {
+  await goToPage(appUrlBase);
+
+  // setup the test fixtures
+  await createCategory(perTestCategoryName);
+  await createLabel(perTestCategoryName, perTestLabelName);
+
+  if (config.withSubset) {
+    await subset({ x1: 0.1, y1: 0.1, x2: 0.8, y2: 0.8 });
+  }
+
+  await waitByClass("autosave-complete");
+}
+
+describe.each([
+  { withSubset: true, tag: "subset" },
+  { withSubset: false, tag: "whole" },
+])("annotations", (config) => {
+  test("create a category", async () => {
+    await setup(config);
+
+    const categoryName = `category-created-${config.tag}`;
+
+    await assertCategoryDoesNotExist(categoryName);
+
+    await createCategory(categoryName);
+
+    await assertCategoryExists(categoryName);
+  });
+
+  test("delete a category", async () => {
+    await setup(config);
+
+    await deleteCategory(perTestCategoryName);
+    await assertCategoryDoesNotExist(perTestCategoryName);
+  });
+
+  test("rename a category", async () => {
+    await setup(config);
+
+    const newCategoryName = `NEW-${config.tag}`;
+
+    await renameCategory(perTestCategoryName, newCategoryName);
+    await assertCategoryDoesNotExist(perTestCategoryName);
+    await assertCategoryExists(newCategoryName);
+  });
+
+  test("create a label", async () => {
+    await setup(config);
+
+    const labelName = `new-label-${config.tag}`;
+
+    await assertLabelDoesNotExist(perTestCategoryName, labelName);
+
+    await createLabel(perTestCategoryName, labelName);
+
+    await assertLabelExists(perTestCategoryName, labelName);
+  });
+
+  test("delete a label", async () => {
+    await setup(config);
+
+    await deleteLabel(perTestCategoryName, perTestLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName, perTestLabelName);
+  });
+
+  test("rename a label", async () => {
+    await setup(config);
+
+    const newLabelName = "my-cool-new-label";
+
+    await assertLabelDoesNotExist(perTestCategoryName, newLabelName);
+    await renameLabel(perTestCategoryName, perTestLabelName, newLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName, perTestLabelName);
+    await assertLabelExists(perTestCategoryName, newLabelName);
+  });
+
+  test("check cell count for a label loaded from file", async () => {
+    await setup(config);
+
+    const duplicateCategoryName = "duplicate";
+    await duplicateCategory(duplicateCategoryName);
+
+    await page.reload({ waitUntil: ["networkidle0", "domcontentloaded"] });
+
+    const firstCategoryExpandIcon = await expect(page).toMatchElement(
+      getTestClass("category-expand")
+    );
+
+    await firstCategoryExpandIcon.click();
+
+    const expectedCategoryRow = await expect(page).toMatchElement(
+      getTestClass("categorical-row")
+    );
+    const expectedLabelName = await getInnerText(
+      expectedCategoryRow,
+      "categorical-value"
+    );
+    const expectedLabelCount = await getInnerText(
+      expectedCategoryRow,
+      "categorical-value-count"
+    );
+
+    await expandCategory(duplicateCategoryName);
+
+    const expectedCategory = await expect(page).toMatchElement(
+      getTestClass("category")
+    );
+
+    const actualCategoryRow = await expect(expectedCategory).toMatchElement(
+      getTestClass("categorical-row")
+    );
+    const actualLabelName = await getInnerText(
+      actualCategoryRow,
+      "categorical-value"
+    );
+    const actualLabelCount = await getInnerText(
+      actualCategoryRow,
+      "categorical-value-count"
+    );
+
+    expect(actualLabelName).toBe(expectedLabelName);
+    expect(actualLabelCount).toBe(expectedLabelCount);
+
+    async function getInnerText(element, className) {
+      return element.$eval(getTestClass(className), (node) => node?.innerText);
+    }
+  });
+
+  test("assign cells to a label", async () => {
+    await setup(config);
+
+    await expandCategory(perTestCategoryName);
+
+    const lassoSelection = await calcDragCoordinates(
+      "layout-graph",
+      data.categoryLabel.lasso["coordinates-as-percent"]
+    );
+
+    await drag("layout-graph", lassoSelection.start, lassoSelection.end, true);
+    await waitByID("lasso-element", { visible: true });
+    await clickOn(`${perTestCategoryName}:${perTestLabelName}:see-actions`);
+    await clickOn(
+      `${perTestCategoryName}:${perTestLabelName}:add-current-selection-to-this-label`
+    );
+
+    const result = await waitByID(
+      `categorical-value-count-${perTestCategoryName}-${perTestLabelName}`
+    );
+
+    expect(await result.evaluate((node) => node.innerText)).toBe(
+      data.categoryLabel.newCount.bySubsetConfig[config.withSubset]
+    );
+  });
+
+  test("undo/redo category creation", async () => {
+    await setup(config);
+
+    const categoryName = `category-created-undo-${config.tag}`;
+
+    await assertCategoryDoesNotExist(categoryName);
+    await createCategory(categoryName);
+    await assertCategoryExists(categoryName);
+    await clickOn("undo");
+    await assertCategoryDoesNotExist(categoryName);
+    await clickOn("redo");
+    await assertCategoryExists(categoryName);
+  });
+
+  test("undo/redo category deletion", async () => {
+    await setup(config);
+
+    const categoryName = `category-deleted-undo-${config.tag}`;
+
+    await createCategory(categoryName);
+    await assertCategoryExists(categoryName);
+    await deleteCategory(categoryName);
+    await assertCategoryDoesNotExist(categoryName);
+    await clickOn("undo");
+    await assertCategoryExists(categoryName);
+    await clickOn("redo");
+    await assertCategoryDoesNotExist(categoryName);
+  });
+
+  test("undo/redo category rename", async () => {
+    await setup(config);
+
+    const newCategoryName = `category-renamed-undo-${config.tag}`;
+
+    await assertCategoryDoesNotExist(newCategoryName);
+    await renameCategory(perTestCategoryName, newCategoryName);
+    await assertCategoryExists(newCategoryName);
+    await assertCategoryDoesNotExist(perTestCategoryName);
+    await clickOn("undo");
+    await assertCategoryExists(perTestCategoryName);
+    await assertCategoryDoesNotExist(newCategoryName);
+    await clickOn("redo");
+    await assertCategoryExists(newCategoryName);
+    await assertCategoryDoesNotExist(perTestCategoryName);
+  });
+
+  test("undo/redo label creation", async () => {
+    await setup(config);
+
+    const labelName = `label-created-undo-${config.tag}`;
+
+    await assertLabelDoesNotExist(perTestCategoryName, labelName);
+    await createLabel(perTestCategoryName, labelName);
+    await assertLabelExists(perTestCategoryName, labelName);
+    await clickOn("undo");
+    await assertLabelDoesNotExist(perTestCategoryName);
+    await clickOn("redo");
+    await assertLabelExists(perTestCategoryName, labelName);
+  });
+
+  test("undo/redo label deletion", async () => {
+    await setup(config);
+
+    await deleteLabel(perTestCategoryName, perTestLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName);
+    await clickOn("undo");
+    await assertLabelExists(perTestCategoryName, perTestLabelName);
+    await clickOn("redo");
+    await assertLabelDoesNotExist(perTestCategoryName);
+  });
+
+  test("undo/redo label rename", async () => {
+    await setup(config);
+
+    const newLabelName = `label-renamed-undo-${config.tag}`;
+
+    await assertLabelDoesNotExist(perTestCategoryName, newLabelName);
+    await renameLabel(perTestCategoryName, perTestLabelName, newLabelName);
+    await assertLabelExists(perTestCategoryName, newLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName, perTestLabelName);
+    await clickOn("undo");
+    await assertLabelExists(perTestCategoryName, perTestLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName, newLabelName);
+    await clickOn("redo");
+    await assertLabelExists(perTestCategoryName, newLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName, perTestLabelName);
+  });
+
+  test("stacked bar graph renders", async () => {
+    await setup(config);
+
+    await expandCategory(perTestCategoryName);
+
+    await clickOn(`colorby-louvain`);
+
+    const labels = await getAllByClass("categorical-row");
+
+    const result = await Promise.all(
+      labels.map((label) => {
+        return page.evaluate((element) => {
+          return element.outerHTML;
+        }, label);
+      })
+    );
+
+    expect(result).toMatchSnapshot();
+  });
+
+  test("truncate midpoint whitespace", async () => {
+    await setup(config);
+    const newLabelName = "123 456";
+    await renameLabel(perTestCategoryName, perTestLabelName, newLabelName);
+    const value = await waitByID(
+      `categorical-value-${perTestCategoryName}-${newLabelName}`
+    );
+    const result = await page.evaluate((elem) => elem.outerHTML, value);
+    expect(result).toMatchSnapshot();
+  });
+
+  test("truncate single character", async () => {
+    await setup(config);
+    const newLabelName = "T";
+    await renameLabel(perTestCategoryName, perTestLabelName, newLabelName);
+    const value = await waitByID(
+      `categorical-value-${perTestCategoryName}-${newLabelName}`
+    );
+    const result = await page.evaluate((elem) => elem.outerHTML, value);
+    expect(result).toMatchSnapshot();
+  });
+
+  async function assertCategoryExists(categoryName) {
+    const handle = await waitByID(`${categoryName}:category-label`);
+
+    const result = await handle.evaluate((node) =>
+      node.getAttribute("aria-label")
+    );
+
+    return expect(result).toBe(categoryName);
+  }
+
+  async function assertLabelExists(categoryName, labelName) {
+    await expect(page).toMatchElement(
+      getTestId(`${categoryName}:category-expand`)
+    );
+
+    await expandCategory(categoryName);
+
+    const previous = await waitByID(
+      `categorical-value-${categoryName}-${labelName}`
+    );
+
+    expect(
+      await previous.evaluate((node) => node.getAttribute("aria-label"))
+    ).toBe(labelName);
+  }
+
+  async function assertLabelDoesNotExist(categoryName, labelName) {
+    await expandCategory(categoryName);
+    const result = await page.$(
+      `[data-testid='categorical-value-${categoryName}-${labelName}']`
+    );
+    expect(result).toBeNull();
+  }
+});

client/__tests__/e2e/e2eJestConfig.json

@@ -0,0 +1,10 @@
+{
+  "testRunner": "jest-circus/runner",
+  "preset": "jest-puppeteer",
+  "testMatch": ["**/__tests__/**/?(*.)(spec|test).js?(x)"],
+  "setupFiles": ["../setupMissingGlobals.js"],
+  "setupFilesAfterEnv": ["expect-puppeteer", "./puppeteer.setup.js"],
+  "globalSetup": "jest-environment-puppeteer/setup",
+  "globalTeardown": "jest-environment-puppeteer/teardown",
+  "testEnvironment": "./screenshot_env.js"
+}

client/__tests__/e2e/puppeteer.setup.js

@@ -0,0 +1,58 @@
+/**
+ * `client/jest-puppeteer.config.js` is for configuring Puppeteer's launch config options
+ * `client/__tests__/e2e/puppeteer.setup.js` is for configuring `jest`, `browser`,
+ * and `page` objects
+ */
+
+import { setDefaultOptions } from "expect-puppeteer";
+import { isDebug, isDev } from "./config";
+import * as ENV_DEFAULT from "../../../environment.default.json";
+
+// (thuang): This is the max time a test can take to run.
+// Since when debugging, we run slowMo and !headless, this means
+// a test can take more time to finish, so we don't want
+// jest to shut off the test too soon
+jest.setTimeout(2 * 60 * 1000);
+setDefaultOptions({ timeout: 20 * 1000 });
+
+jest.retryTimes(ENV_DEFAULT.RETRY_ATTEMPTS);
+
+(async () => {
+  const userAgent = await browser.userAgent();
+  await page.setUserAgent(`${userAgent}bot`);
+
+  await page._client.send("Animation.setPlaybackRate", { playbackRate: 12 });
+
+  page.on("pageerror", (err) => {
+    throw new Error(`Console error: ${err}`);
+  });
+
+  page.on("error", (err) => {
+    throw new Error(`Console error: ${err}`);
+  });
+
+  page.on("console", async (msg) => {
+    if (isDev || isDebug) {
+      // If there is a console.error but an error is not thrown, this will ensure the test fails
+      console.log(`PAGE LOG: ${msg.text()}`);
+      if (msg.type() === "error") {
+        // TODO: chromium does not currently support the CSP directive on the
+        // line below, so we swallow this error. Remove this when the test
+        // suite uses a browser version that supports this directive.
+        if (
+          msg.text() ===
+          "Unrecognized Content-Security-Policy directive 'require-trusted-types-for'.\n"
+        ) {
+          return;
+        }
+        const errorMsgText = await Promise.all(
+          // TODO can we do this without internal properties?
+          msg.args().map((arg) => arg._remoteObject.description)
+        );
+        throw new Error(`Console error: ${errorMsgText}`);
+      }
+    }
+  });
+})().catch((error) => {
+  console.error("puppeteer.setup.js error", error);
+});

client/__tests__/e2e/puppeteerUtils.js

@@ -0,0 +1,131 @@
+/* eslint-disable no-await-in-loop -- await in loop is needed to emulate sequential user actions */
+export function getTestId(id) {
+  return `[data-testid='${id}']`;
+}
+
+export function getTestClass(className) {
+  return `[data-testclass='${className}']`;
+}
+
+export async function waitByID(testId, props = {}) {
+  return page.waitForSelector(getTestId(testId), props);
+}
+
+export async function waitByClass(testClass, props = {}) {
+  await page.waitForSelector(`[data-testclass='${testClass}']`, props);
+}
+
+export async function waitForAllByIds(testIds) {
+  await Promise.all(
+    testIds.map((testId) => page.waitForSelector(getTestId(testId)))
+  );
+}
+
+export async function getAllByClass(testClass) {
+  return page.$$(`[data-testclass=${testClass}]`);
+}
+
+export async function typeInto(testId, text) {
+  // blueprint's  typeahead is treating typing weird, clicking & waiting first solves this
+  // only works for text without special characters
+  await waitByID(testId);
+  const selector = getTestId(testId);
+  // type ahead can be annoying if you don't pause before you type
+  await page.click(selector);
+  await page.waitFor(200);
+  await page.type(selector, text);
+}
+
+export async function clearInputAndTypeInto(testId, text) {
+  await waitByID(testId);
+  const selector = getTestId(testId);
+  // only works for text without special characters
+  // type ahead can be annoying if you don't pause before you type
+  await page.click(selector);
+  await page.waitFor(200);
+  // select all
+  await page.click(selector, { clickCount: 3 });
+  await page.keyboard.press("Backspace");
+  await page.type(selector, text);
+}
+
+export async function clickOn(testId, options = {}) {
+  await expect(page).toClick(getTestId(testId), options);
+}
+
+/**
+ * (thuang): There are times when Puppeteer clicks on a button and the page doesn't respond.
+ * So I added clickOnUntil() to retry clicking until a given condition is met.
+ */
+export async function clickOnUntil(testId, assert) {
+  const MAX_RETRY = 10;
+  const WAIT_FOR_MS = 200;
+
+  let retry = 0;
+
+  while (retry < MAX_RETRY) {
+    try {
+      await clickOn(testId);
+      await assert();
+
+      break;
+    } catch (error) {
+      retry += 1;
+
+      await page.waitFor(WAIT_FOR_MS);
+    }
+  }
+
+  if (retry === MAX_RETRY) {
+    throw Error("clickOnUntil() assertion failed!");
+  }
+}
+
+export async function getOneElementInnerHTML(selector, options = {}) {
+  await page.waitForSelector(selector, options);
+
+  return page.$eval(selector, (el) => el.innerHTML);
+}
+
+export async function getOneElementInnerText(selector) {
+  expect(page).toMatchElement(selector);
+
+  return page.$eval(selector, (el) => el.innerText);
+}
+
+export async function getElementCoordinates(testId) {
+  return page.$eval(getTestId(testId), (elem) => {
+    const { left, top } = elem.getBoundingClientRect();
+    return [left, top];
+  });
+}
+
+async function clickTermsOfService() {
+  if (!(await isElementPresent(getTestId("tos-cookies-accept")))) return;
+
+  await clickOn("tos-cookies-accept");
+}
+
+async function nameNewAnnotation() {
+  if (await isElementPresent(getTestId("annotation-dialog"))) {
+    await typeInto("new-annotation-name", "ignoreE2E");
+    await clickOn("submit-annotation");
+
+    // wait for the page to load
+    await waitByClass("autosave-complete");
+  }
+}
+
+export async function goToPage(url) {
+  await page.goto(url, {
+    waitUntil: "networkidle0",
+  });
+
+  await nameNewAnnotation();
+  await clickTermsOfService();
+}
+
+export async function isElementPresent(selector, options) {
+  return Boolean(await page.$(selector, options));
+}
+/* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/screenshot_env.js

@@ -0,0 +1,36 @@
+const PuppeteerEnvironment = require("jest-environment-puppeteer");
+require("jest-circus");
+const ENV_DEFAULT = require("../../../environment.default.json");
+
+const takeScreenshot = require("./takeScreenshot");
+
+class ScreenshotEnvironment extends PuppeteerEnvironment {
+  async handleTestEvent(event, state) {
+    if (["test_start", "test_done"].includes(event.name)) {
+      console.log("------------------event name:\n", event.name);
+      console.log("~~~~ Current test errors\n", new Date(), event.test.errors);
+      console.log("~~~~ Current test\n", new Date(), event.test);
+    }
+
+    if (event.name === "error") {
+      console.log("error event:", JSON.stringify(event));
+    }
+
+    if (event.name === "test_fn_failure" || event.name === "hook_failure") {
+      console.log("------------------event name:\n", event.name);
+      console.log(">>>> Current state\n", new Date(), state);
+      console.log("===> Failure event\n", new Date(), event);
+
+      // (thuang): We only want to take screenshot on the last try
+      if (
+        state.currentlyRunningTest.invocations <= ENV_DEFAULT.RETRY_ATTEMPTS
+      ) {
+        return;
+      }
+
+      await takeScreenshot(state.currentlyRunningTest.name, this.global.page);
+    }
+  }
+}
+
+module.exports = ScreenshotEnvironment;

client/__tests__/e2e/takeScreenshot.js

@@ -0,0 +1,17 @@
+function toFilename(name) {
+  return name.replace(/[^a-z0-9.-]+/gi, "-");
+}
+
+async function takeScreenshot(currentTestName, page) {
+  const testName = toFilename(currentTestName);
+
+  // Take a screenshot at the point of failure
+  const date = new Date().toISOString();
+  const screenshotName = `${date}-${testName}.png`;
+
+  await page.screenshot({
+    path: `./__tests__/screenshots/ignoreE2E-screenshot-${screenshotName}`,
+  });
+}
+
+module.exports = takeScreenshot;

client/__tests__/reducers/cascade.test.js

@@ -0,0 +1,48 @@
+import cascadeReducers from "../../src/reducers/cascade";
+
+describe("create", () => {
+  test("from Array", () => {
+    expect(cascadeReducers([["foo", () => 0]])).toBeInstanceOf(Function);
+  });
+
+  test("from Map", () => {
+    expect(cascadeReducers(new Map([["foo", () => 0]]))).toBeInstanceOf(
+      Function
+    );
+  });
+});
+
+describe("cascade", () => {
+  test("expected arguments provided & cascade ordering", () => {
+    const topLevelState = {};
+    const topLevelAction = { type: "test" };
+
+    const reducer = cascadeReducers([
+      [
+        "foo",
+        (currentState, action, nextSharedState, prevSharedState) => {
+          expect(currentState).toBeUndefined();
+          expect(action).toEqual(topLevelAction);
+          expect(nextSharedState).toStrictEqual({});
+          expect(prevSharedState).toBe(topLevelState);
+          return 0;
+        },
+      ],
+      [
+        "bar",
+        (currentState, action, nextSharedState, prevSharedState) => {
+          expect(currentState).toBeUndefined();
+          expect(action).toEqual(topLevelAction);
+          expect(nextSharedState).toStrictEqual({ foo: 0 });
+          expect(prevSharedState).toBe(topLevelState);
+          return 99;
+        },
+      ],
+    ]);
+
+    const nextState = reducer(topLevelState, topLevelAction);
+    expect(nextState).toStrictEqual({ foo: 0, bar: 99 });
+    expect(topLevelState).toStrictEqual({});
+    expect(topLevelAction).toStrictEqual({ type: "test" });
+  });
+});

client/__tests__/reducers/undoable.test.js

@@ -0,0 +1,87 @@
+import undoable from "../../src/reducers/undoable";
+
+describe("create", () => {
+  test("no keys", () => {
+    expect(() => undoable(() => {})).toThrow();
+    expect(() => undoable(() => {}, null)).toThrow();
+    expect(() => undoable(() => {}, [])).toThrow();
+    expect(() => undoable(() => {}, [], {})).toThrow();
+  });
+
+  test("simple", () => {
+    expect(undoable(() => {}, ["foo"])).toBeInstanceOf(Function);
+    expect(undoable(() => {}, ["foo"], {})).toBeInstanceOf(Function);
+  });
+
+  test("handles undefined initial state", () => {
+    expect(
+      undoable(() => {}, ["a"])(undefined, { type: "test" })
+    ).toMatchObject({});
+  });
+});
+
+describe("undo", () => {
+  test("expected state modifications", () => {
+    const initialState = { a: 0, b: 1000 };
+    const reducer = (state) => {
+      return { a: state.a + 1, b: state.b + 1 };
+    };
+    const undoableReducer = undoable(reducer, ["a"]);
+
+    const s1 = undoableReducer(initialState, { type: "test" });
+    expect(s1).toMatchObject({ a: 1, b: 1001 });
+
+    // test that only specified keys are undone
+    const s2 = undoableReducer(s1, { type: "@@undoable/undo" });
+    expect(s2).toMatchObject({ a: 0, b: 1001 });
+
+    // test backstop when no more history
+    const s3 = undoableReducer(s2, { type: "@@undoable/undo" });
+    expect(s3).toMatchObject({ a: 0, b: 1001 });
+  });
+});
+
+describe("redo", () => {
+  const initialState = { a: 0, b: 1000 };
+  const reducer = (state) => {
+    return { a: state.a + 1, b: state.b + 1 };
+  };
+  let UR;
+
+  beforeEach(() => {
+    UR = undoable(reducer, ["a"]);
+  });
+
+  test("expected state modifications", () => {
+    const s1 = UR(initialState, { type: "test" });
+    expect(s1).toMatchObject({ a: 1, b: 1001 });
+
+    // verify undo->redo reverts state.
+    const s2 = UR(UR(s1, { type: "@@undoable/undo" }), {
+      type: "@@undoable/redo",
+    });
+    expect(s2).toMatchObject({ a: 1, b: 1001 });
+
+    // verify backstop when no redo future
+    const s3 = UR(s2, { type: "@@undoable/redo" });
+    expect(s3).toMatchObject({ a: 1, b: 1001 });
+  });
+
+  test("history cleared", () => {
+    // verify future cleared upon a normal state transition
+    const s1 = UR(initialState, { type: "test" });
+    expect(s1).toMatchObject({ a: 1, b: 1001 });
+    const s2 = UR(s1, { type: "@@undoable/undo" });
+    expect(s2).toMatchObject({ a: 0, b: 1001 });
+    const s3 = UR(s2, { type: "test" });
+    expect(s3).toMatchObject({ a: 1, b: 1002 });
+    const s4 = UR(s3, { type: "@@undoable/redo" });
+    expect(s4).toMatchObject({ a: 1, b: 1002 });
+  });
+});
+
+/*
+TODO:
+- historyLimit is enforced
+- action filters
+*/

client/__tests__/screenshots/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

client/__tests__/setupMissingGlobals.js

@@ -0,0 +1,9 @@
+/*
+Define globals which are present in the client, but not in node (and therefore not in
+the jest test environment).
+*/
+
+import { TextDecoder, TextEncoder } from "util";
+
+global.TextDecoder = TextDecoder;
+global.TextEncoder = TextEncoder;

client/__tests__/util/actionHelpers.test.js

@@ -0,0 +1,63 @@
+import { rangeEncodeIndices } from "../../src/util/actionHelpers";
+
+describe("rangeEncodeIndices", () => {
+  test("small array edge cases", () => {
+    expect(rangeEncodeIndices([])).toMatchObject([]);
+    expect(rangeEncodeIndices([1])).toMatchObject([1]);
+    expect(rangeEncodeIndices([1, 99])).toMatchObject([1, 99]);
+    expect(rangeEncodeIndices([99, 1])).toMatchObject([1, 99]);
+    expect(rangeEncodeIndices([1, 100, 4])).toMatchObject([1, 4, 100]);
+  });
+
+  test("sorted flag", () => {
+    expect(rangeEncodeIndices([1, 9, 432], 10, true)).toMatchObject([
+      1,
+      9,
+      432,
+    ]);
+    expect(rangeEncodeIndices([1, 9, 432], 10, false)).toMatchObject([
+      1,
+      9,
+      432,
+    ]);
+    expect(
+      rangeEncodeIndices([0, 1, 2, 3, 9, 10, 432], 2, true)
+    ).toMatchObject([[0, 3], [9, 10], 432]);
+    expect(
+      rangeEncodeIndices([0, 1, 2, 3, 9, 10, 432], 2, false)
+    ).toMatchObject([[0, 3], [9, 10], 432]);
+  });
+
+  test("begin or end edge cases", () => {
+    expect(
+      rangeEncodeIndices([3, 4, 5, 6, 7, 10, 11, 12, 99], 3, false)
+    ).toMatchObject([[3, 7], [10, 12], 99]);
+    expect(
+      rangeEncodeIndices([3, 4, 5, 6, 7, 10, 11, 12], 3, false)
+    ).toMatchObject([
+      [3, 7],
+      [10, 12],
+    ]);
+    expect(
+      rangeEncodeIndices([0, 3, 4, 5, 6, 7, 10, 11, 12], 3, false)
+    ).toMatchObject([0, [3, 7], [10, 12]]);
+    expect(
+      rangeEncodeIndices([0, 3, 4, 5, 6, 7, 10, 11, 12, 99], 3, false)
+    ).toMatchObject([0, [3, 7], [10, 12], 99]);
+  });
+
+  test("minRangeLength", () => {
+    expect(
+      rangeEncodeIndices([3, 4, 5, 6, 7, 10, 11, 12, 99], 4, false)
+    ).toMatchObject([[3, 7], 10, 11, 12, 99]);
+    expect(
+      rangeEncodeIndices([3, 4, 5, 6, 7, 10, 11, 12], 4, false)
+    ).toMatchObject([[3, 7], 10, 11, 12]);
+    expect(
+      rangeEncodeIndices([0, 3, 4, 5, 6, 7, 10, 11, 12], 4, false)
+    ).toMatchObject([0, [3, 7], 10, 11, 12]);
+    expect(
+      rangeEncodeIndices([0, 3, 4, 5, 6, 7, 10, 11, 12, 99], 4, false)
+    ).toMatchObject([0, [3, 7], 10, 11, 12, 99]);
+  });
+});

client/__tests__/util/annoMatrix/annoMatrix.test.js

@@ -0,0 +1,316 @@
+// these TWO statements MUST be first in the file, before any other imports
+import { enableFetchMocks } from "jest-fetch-mock";
+import * as serverMocks from "./serverMocks";
+// OK, continue on!
+
+import {
+  AnnoMatrixLoader,
+  clip,
+  isubset,
+  isubsetMask,
+} from "../../../src/annoMatrix";
+import { Dataframe } from "../../../src/util/dataframe";
+
+enableFetchMocks();
+
+describe("AnnoMatrix", () => {
+  let annoMatrix;
+
+  beforeEach(async () => {
+    fetch.resetMocks(); // reset all fetch mocking state
+    annoMatrix = new AnnoMatrixLoader(
+      serverMocks.baseDataURL,
+      serverMocks.schema.schema
+    );
+  });
+
+  describe("basics", () => {
+    test("annomatrix static checks", () => {
+      expect(annoMatrix).toBeDefined();
+      expect(annoMatrix.schema).toMatchObject(serverMocks.schema.schema);
+      expect(annoMatrix.nObs).toEqual(serverMocks.schema.schema.dataframe.nObs);
+      expect(annoMatrix.nVar).toEqual(serverMocks.schema.schema.dataframe.nVar);
+      expect(annoMatrix.isView).toBeFalsy();
+      expect(annoMatrix.viewOf).toBeUndefined();
+      expect(annoMatrix.rowIndex).toBeDefined();
+    });
+
+    test("simple single column fetch", async () => {
+      fetch.once(serverMocks.annotationsObs(["name_0"]));
+
+      const df = await annoMatrix.fetch("obs", "name_0");
+      expect(df).toBeInstanceOf(Dataframe);
+      expect(df.colIndex.labels()).toEqual(["name_0"]);
+      expect(df.dims).toEqual([annoMatrix.nObs, 1]);
+    });
+
+    test("simple multi column fetch", async () => {
+      fetch
+        .once(serverMocks.annotationsObs(["name_0"]))
+        .once(serverMocks.annotationsObs(["n_genes"]));
+
+      await expect(
+        annoMatrix.fetch("obs", ["name_0", "n_genes"])
+      ).resolves.toBeInstanceOf(Dataframe);
+    });
+
+    describe("fetch from field", () => {
+      const getLastTwo = async (field) => {
+        const columnNames = annoMatrix.getMatrixColumns(field).slice(-2);
+        fetch.mockResponses(...columnNames.map(() => serverMocks.responder));
+        await expect(
+          annoMatrix.fetch(field, columnNames)
+        ).resolves.toBeInstanceOf(Dataframe);
+      };
+
+      test("obs", async () => getLastTwo("obs"));
+      test("var", async () => getLastTwo("var"));
+      test("emb", async () => getLastTwo("emb"));
+    });
+
+    test("fetch - test all query forms", async () => {
+      // single string is a column name
+      fetch.once(serverMocks.annotationsObs(["n_genes"]));
+      await expect(annoMatrix.fetch("obs", "n_genes")).resolves.toBeInstanceOf(
+        Dataframe
+      );
+
+      // array of column names, expecting n_genes to be cached.
+      fetch.once(serverMocks.annotationsObs(["percent_mito"]));
+      await expect(
+        annoMatrix.fetch("obs", ["n_genes", "percent_mito"])
+      ).resolves.toBeInstanceOf(Dataframe);
+
+      // more complex value filter query, enumerated
+      fetch.once(serverMocks.responder);
+      await expect(
+        annoMatrix.fetch("X", {
+          field: "var",
+          column: annoMatrix.schema.annotations.var.index,
+          value: "TYMP",
+        })
+      ).resolves.toBeInstanceOf(Dataframe);
+
+      // more complex value filter query, range
+      const varIndex = annoMatrix.schema.annotations.var.index;
+      fetch
+        .once(
+          serverMocks.withExpected("/data/var", [[`var:${varIndex}`, "SUMO3"]])
+        )
+        .once(
+          serverMocks.withExpected("/data/var", [[`var:${varIndex}`, "TYMP"]])
+        );
+      await expect(
+        annoMatrix.fetch("X", [
+          {
+            field: "var",
+            column: varIndex,
+            value: "SUMO3",
+          },
+          {
+            field: "var",
+            column: varIndex,
+            value: "TYMP",
+          },
+        ])
+      ).resolves.toBeInstanceOf(Dataframe);
+      // XXX inspect the wherecache?
+    });
+
+    test("push and pop views", async () => {
+      const am1 = clip(annoMatrix, 0.1, 0.9);
+      expect(am1.viewOf).toBe(annoMatrix);
+      expect(am1.nObs).toEqual(annoMatrix.nObs);
+      expect(am1.nVar).toEqual(annoMatrix.nVar);
+      expect(am1.rowIndex).toBe(annoMatrix.rowIndex);
+
+      const am2 = clip(annoMatrix, 0.1, 0.9);
+      expect(am2.viewOf).toBe(annoMatrix);
+      expect(am2).not.toBe(am1);
+      expect(am2.rowIndex).toBe(annoMatrix.rowIndex);
+    });
+
+    test("schema accessors", () => {
+      expect(annoMatrix.getMatrixFields()).toEqual(
+        expect.arrayContaining(["X", "obs", "emb", "var"])
+      );
+      expect(annoMatrix.getMatrixColumns("obs")).toEqual(
+        expect.arrayContaining(["name_0", "n_genes", "louvain"])
+      );
+      expect(annoMatrix.getColumnSchema("emb", "umap")).toEqual({
+        name: "umap",
+        dims: ["umap_0", "umap_1"],
+        type: "float32",
+      });
+      expect(annoMatrix.getColumnDimensions("emb", "umap")).toEqual([
+        "umap_0",
+        "umap_1",
+      ]);
+    });
+
+    /*
+  	test the mask & label access to subset via isubset and isubsetMask
+  	*/
+    test("isubset", async () => {
+      const rowList = [0, 10];
+      const rowMask = new Uint8Array(annoMatrix.nObs);
+      for (let i = 0; i < rowList.length; i += 1) {
+        rowMask[rowList[i]] = 1;
+      }
+
+      const am1 = isubset(annoMatrix, rowList);
+      const am2 = isubsetMask(annoMatrix, rowMask);
+      expect(am1).not.toBe(am2);
+      expect(am1.nObs).toEqual(2);
+      expect(am1.nObs).toEqual(am2.nObs);
+      expect(am1.nVar).toEqual(am2.nVar);
+
+      fetch
+        .once(serverMocks.annotationsObs(["n_genes"]))
+        .once(serverMocks.annotationsObs(["n_genes"]));
+      const ng1 = await am1.fetch("obs", "n_genes");
+      const ng2 = await am2.fetch("obs", "n_genes");
+      expect(ng1).toHaveLength(ng2.length);
+      expect(ng1.colIndex.labels()).toEqual(ng2.colIndex.labels());
+      expect(ng1.col("n_genes").asArray()).toEqual(
+        ng2.col("n_genes").asArray()
+      );
+    });
+  });
+
+  describe("add/drop column", () => {
+    async function addDrop(base) {
+      expect(base.getMatrixColumns("obs")).not.toContain("foo");
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(base.fetch("obs", "foo")).rejects.toThrow(
+        "unknown column name"
+      );
+
+      /* add */
+      const am1 = base.addObsColumn(
+        { name: "foo", type: "float32", writable: true },
+        Float32Array,
+        0
+      );
+      expect(base.getMatrixColumns("obs")).not.toContain("foo");
+      expect(am1.getMatrixColumns("obs")).toContain("foo");
+      const foo = await am1.fetch("obs", "foo");
+      expect(foo).toBeDefined();
+      expect(foo).toBeInstanceOf(Dataframe);
+      expect(foo).toHaveLength(am1.nObs);
+      expect(foo.col("foo").asArray()).toEqual(
+        new Float32Array(am1.nObs).fill(0)
+      );
+
+      /* drop */
+      const am2 = am1.dropObsColumn("foo");
+      expect(base.getMatrixColumns("obs")).not.toContain("foo");
+      expect(am2.getMatrixColumns("obs")).not.toContain("foo");
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(am2.fetch("obs", "foo")).rejects.toThrow(
+        "unknown column name"
+      );
+    }
+
+    test("add/drop column, without view", async () => {
+      await addDrop(annoMatrix);
+    });
+
+    test("add/drop column, with view", async () => {
+      const am1 = clip(annoMatrix, 0.1, 0.9);
+      await addDrop(am1);
+
+      const am2 = isubset(am1, [0, 1, 2, 20, 30, 400]);
+      await addDrop(am2);
+
+      const am3 = isubset(annoMatrix, [10, 0, 7, 3]);
+      await addDrop(am3);
+
+      const am4 = clip(am3, 0, 1);
+      await addDrop(am4);
+
+      fetch.mockResponse(serverMocks.responder);
+
+      await am1.fetch("obs", am1.getMatrixColumns("obs"));
+      await am2.fetch("obs", am2.getMatrixColumns("obs"));
+      await am3.fetch("obs", am3.getMatrixColumns("obs"));
+      await am4.fetch("obs", am4.getMatrixColumns("obs"));
+
+      fetch.resetMocks();
+
+      await addDrop(am1);
+      await addDrop(am2);
+      await addDrop(am3);
+      await addDrop(am4);
+    });
+  });
+
+  describe("setObsColumnValues", () => {
+    async function addSetDrop(base) {
+      /* add column */
+      let am = base.addObsColumn(
+        {
+          name: "test",
+          type: "categorical",
+          categories: ["unassigned", "red", "green"],
+          writable: true,
+        },
+        Array,
+        "unassigned"
+      );
+
+      const testVal = await am.fetch("obs", "test");
+      expect(testVal.col("test").asArray()).toEqual(
+        new Array(am.nObs).fill("unassigned")
+      );
+
+      /* set values in column */
+      const whichRows = [1, 2, 10];
+      const am1 = await am.setObsColumnValues("test", whichRows, "yo");
+      const testVal1 = await am1.fetch("obs", "test");
+      const expt = new Array(am1.nObs).fill("unassigned");
+      for (let i = 0; i < whichRows.length; i += 1) {
+        const offset = am1.rowIndex.getOffset(whichRows[i]);
+        expt[offset] = "yo";
+      }
+      expect(testVal1).not.toBe(testVal);
+      expect(testVal1.col("test").asArray()).toEqual(expt);
+      expect(am1.getColumnSchema("obs", "test").type).toBe("categorical");
+      expect(am1.getColumnSchema("obs", "test").categories).toEqual(
+        expect.arrayContaining(["unassigned", "red", "green", "yo"])
+      );
+
+      /* drop column */
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      am = am1.dropObsColumn("test");
+      await expect(am.fetch("obs", "test")).rejects.toThrow(
+        "unknown column name"
+      );
+    }
+
+    test("set, without a view", async () => {
+      await addSetDrop(annoMatrix);
+    });
+
+    test("set, with a view", async () => {
+      const am1 = clip(annoMatrix, 0.1, 0.9);
+      await addSetDrop(am1);
+
+      const am2 = isubset(am1, [0, 1, 2, 10, 20, 30, 400]);
+      await addSetDrop(am2);
+
+      const am3 = isubset(annoMatrix, [10, 1, 0, 30, 2]);
+      await addSetDrop(am3);
+
+      fetch.mockResponse(serverMocks.responder);
+
+      await am1.fetch("obs", am1.getMatrixColumns("obs"));
+      await am2.fetch("obs", am2.getMatrixColumns("obs"));
+      await am3.fetch("obs", am3.getMatrixColumns("obs"));
+
+      await addSetDrop(am1);
+      await addSetDrop(am2);
+      await addSetDrop(am3);
+    });
+  });
+});

client/__tests__/util/annoMatrix/crossfilter.test.js

@@ -0,0 +1,688 @@
+// these TWO statements MUST be first in the file, before any other imports
+import { enableFetchMocks } from "jest-fetch-mock";
+import * as serverMocks from "./serverMocks";
+// OK, continue on!
+
+import obsLouvain from "./louvain.json";
+import obsNGenes from "./n_genes.json";
+import embUmap from "./umap.json";
+
+import {
+  AnnoMatrixLoader,
+  AnnoMatrixObsCrossfilter,
+  isubsetMask,
+} from "../../../src/annoMatrix";
+import { rangeFill } from "../../../src/util/range";
+
+enableFetchMocks();
+
+describe("AnnoMatrixCrossfilter", () => {
+  let annoMatrix;
+  let crossfilter;
+
+  beforeEach(async () => {
+    fetch.resetMocks(); // reset all fetch mocking state
+    annoMatrix = new AnnoMatrixLoader(
+      serverMocks.baseDataURL,
+      serverMocks.schema.schema
+    );
+    crossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
+  });
+
+  test("initial state of crossfilter", () => {
+    const { nObs } = annoMatrix;
+
+    expect(crossfilter).toBeDefined();
+    expect(crossfilter.size()).toEqual(nObs);
+    expect(crossfilter.annoMatrix).toBe(annoMatrix);
+
+    /* by default, everything should be selected, even if no data in cache */
+    expect(crossfilter.countSelected()).toEqual(nObs);
+    expect(crossfilter.allSelectedLabels()).toEqual(
+      rangeFill(new Int32Array(nObs))
+    );
+    expect(crossfilter.allSelectedMask()).toEqual(new Uint8Array(nObs).fill(1));
+    expect(crossfilter.fillByIsSelected(new Uint8Array(nObs), 2, 1)).toEqual(
+      new Uint8Array(nObs).fill(2)
+    );
+  });
+
+  describe("select", () => {
+    /*
+    test the selection state via crossfilter proxy
+    */
+
+    test("select loads index", async () => {
+      /*
+      Select should transparently load/create dimension index.
+
+      Internal dimension names are field/col:col:col..., eg,
+
+        obs:louvain
+        emb:umap_0:umap_1
+
+      */
+      expect(crossfilter.obsCrossfilter.dimensionNames()).toEqual([]);
+      expect(
+        crossfilter.obsCrossfilter.hasDimension("obs/louvain")
+      ).toBeFalsy();
+
+      fetch.once(serverMocks.dataframeResponse(["louvain"], [obsLouvain]));
+      let newCrossfilter = await crossfilter.select("obs", "louvain", {
+        mode: "none",
+      });
+
+      expect(newCrossfilter.countSelected()).toEqual(0);
+      expect(
+        newCrossfilter.obsCrossfilter.hasDimension("obs/louvain")
+      ).toBeTruthy();
+      expect(fetch.mock.calls).toHaveLength(1);
+
+      newCrossfilter = await crossfilter.select("obs", "louvain", {
+        mode: "all",
+      });
+      expect(newCrossfilter.countSelected()).toEqual(annoMatrix.nObs);
+    });
+
+    test("simple column select", async () => {
+      let xfltr;
+
+      fetch.once(serverMocks.dataframeResponse(["louvain"], [obsLouvain]));
+      xfltr = await crossfilter.select("obs", "louvain", {
+        mode: "exact",
+        values: ["NK cells", "B cells"],
+      });
+
+      expect(xfltr).toBeDefined();
+      expect(xfltr.countSelected()).toEqual(496);
+      expect(xfltr.allSelectedMask()).toEqual(
+        Uint8Array.from(
+          obsLouvain.map((val) =>
+            val === "NK cells" || val === "B cells" ? 1 : 0
+          )
+        )
+      );
+      expect(xfltr.allSelectedLabels()).toEqual(
+        Int32Array.from(
+          obsLouvain.reduce((acc, val, idx) => {
+            if (val === "NK cells" || val === "B cells") acc.push(idx);
+            return acc;
+          }, [])
+        )
+      );
+      expect(
+        xfltr.fillByIsSelected(new Uint8Array(annoMatrix.nObs), 3, 1)
+      ).toEqual(
+        Uint8Array.from(
+          obsLouvain.map((val) =>
+            val === "NK cells" || val === "B cells" ? 3 : 1
+          )
+        )
+      );
+
+      const df = await annoMatrix.fetch("obs", "louvain");
+      const values = df.col("louvain").asArray();
+      const selected = xfltr.allSelectedMask();
+      values.every(
+        (val, idx) => !["NK cells", "B cells"].includes(val) !== !selected[idx]
+      );
+
+      fetch.once(
+        serverMocks.dataframeResponse(["n_genes"], [new Int32Array(obsNGenes)])
+      );
+      xfltr = await xfltr.select("obs", "n_genes", {
+        mode: "range",
+        lo: 0,
+        hi: 500,
+        inclusive: false,
+      });
+      expect(xfltr.countSelected()).toEqual(33);
+      expect(xfltr.allSelectedLabels()).toEqual(
+        Int32Array.from(
+          obsNGenes.reduce((acc, val, idx) => {
+            const louvain = obsLouvain[idx];
+            if (
+              val >= 0 &&
+              val < 500 &&
+              (louvain === "NK cells" || louvain === "B cells")
+            )
+              acc.push(idx);
+            return acc;
+          }, [])
+        )
+      );
+
+      xfltr = await xfltr.selectAll();
+      expect(xfltr.countSelected()).toEqual(annoMatrix.nObs);
+    });
+
+    test("join column select", async () => {
+      const varIndex = annoMatrix.schema.annotations.var.index;
+
+      const { nObs } = annoMatrix.schema.dataframe;
+      fetch.once(
+        serverMocks.dataframeResponse(
+          ["TEST"],
+          [rangeFill(new Float32Array(nObs), 0, 0.1)]
+        )
+      );
+
+      const xfltr = await crossfilter.select(
+        "X",
+        {
+          field: "var",
+          column: varIndex,
+          value: "TYMP",
+        },
+        {
+          mode: "range",
+          lo: 0,
+          hi: 50,
+          inclusive: true,
+        }
+      );
+
+      expect(xfltr).toBeDefined();
+      expect(xfltr.countSelected()).toEqual(501);
+
+      const df = await annoMatrix.fetch("X", {
+        field: "var",
+        column: varIndex,
+        value: "TYMP",
+      });
+      const values = df.icol(0).asArray();
+      const selected = xfltr.allSelectedMask();
+      values.every((val, idx) => !(val >= 0 && val <= 50) !== !selected[idx]);
+      expect(selected.reduce((acc, val) => (val ? acc + 1 : acc), 0)).toEqual(
+        xfltr.countSelected()
+      );
+    });
+
+    test("spatial column select", async () => {
+      fetch.once(
+        serverMocks.dataframeResponse(
+          ["umap_0", "umap_1"],
+          [Float32Array.from(embUmap[0]), Float32Array.from(embUmap[1])]
+        )
+      );
+      const xfltr = await crossfilter.select("emb", "umap", {
+        mode: "within-rect",
+        minX: 0,
+        minY: 0,
+        maxX: 0.5,
+        maxY: 0.5,
+      });
+      expect(xfltr.countSelected()).toEqual(16);
+    });
+
+    test("select on subset", async () => {
+      const mask = new Uint8Array(annoMatrix.nObs).fill(0);
+      for (let i = 0; i < mask.length; i += 2) {
+        mask[i] = true;
+      }
+      const annoMatrixSubset = isubsetMask(annoMatrix, mask);
+      expect(annoMatrixSubset.nObs).toEqual(Math.floor(annoMatrix.nObs / 2));
+
+      let xfltr = new AnnoMatrixObsCrossfilter(annoMatrixSubset);
+      expect(xfltr.countSelected()).toEqual(annoMatrixSubset.nObs);
+
+      fetch.once(serverMocks.dataframeResponse(["louvain"], [obsLouvain]));
+      xfltr = await xfltr.select("obs", "louvain", {
+        mode: "exact",
+        values: ["NK cells", "B cells"],
+      });
+
+      expect(xfltr).toBeDefined();
+      expect(xfltr.countSelected()).toEqual(240);
+
+      const df = await annoMatrixSubset.fetch("obs", "louvain");
+      const values = df.col("louvain").asArray();
+      const selected = xfltr.allSelectedMask();
+      values.every(
+        (val, idx) => !["NK cells", "B cells"].includes(val) !== !selected[idx]
+      );
+    });
+
+    test("select catches errors", async () => {
+      await expect(crossfilter.select("NADA", "foo")).rejects.toThrow(
+        "Unknown field name"
+      );
+
+      await expect(crossfilter.select("var", "foo")).rejects.toThrow(
+        "unable to obsSelect upon the var dimension"
+      );
+
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(crossfilter.select("obs", "foo")).rejects.toThrow(
+        "unknown column name"
+      );
+    });
+  });
+
+  describe("mutate matrix", () => {
+    /*
+    test the matrix mutators via crossfilter proxy
+    */
+    async function helperAddTestCol(cf, colName, colSchema = null) {
+      expect(
+        cf.annoMatrix.getMatrixColumns("obs").includes(colName)
+      ).toBeFalsy();
+
+      if (colSchema === null) {
+        colSchema = {
+          name: colName,
+          type: "categorical",
+          categories: ["toasty"],
+        };
+      }
+      colSchema.name = colName;
+      const initValue = colSchema.categories[0];
+      const xfltr = cf.addObsColumn(colSchema, Array, initValue);
+      expect(
+        xfltr.annoMatrix.schema.annotations.obs.columns.filter(
+          (v) => v.name === colName
+        )
+      ).toHaveLength(1);
+      const df = await xfltr.annoMatrix.fetch("obs", colName);
+      expect(df.hasCol(colName)).toBeTruthy();
+      return xfltr;
+    }
+
+    test("addObsColumn", async () => {
+      expect(crossfilter.countSelected()).toBe(annoMatrix.nObs);
+      expect(
+        crossfilter.annoMatrix.getMatrixColumns("obs").includes("foo")
+      ).toBeFalsy();
+      const xfltr = crossfilter.addObsColumn(
+        { name: "foo", type: "categorical", categories: ["A"] },
+        Array,
+        "A"
+      );
+
+      // check schema updates correctly.
+      expect(xfltr.countSelected()).toBe(annoMatrix.nObs);
+      expect(
+        xfltr.annoMatrix.getMatrixColumns("obs").includes("foo")
+      ).toBeTruthy();
+      expect(xfltr.annoMatrix.schema.annotations.obsByName.foo).toMatchObject({
+        name: "foo",
+        type: "categorical",
+      });
+      expect(
+        xfltr.annoMatrix.schema.annotations.obs.columns.filter(
+          (v) => v.name === "foo"
+        )
+      ).toHaveLength(1);
+
+      // check data update.
+      const df = await xfltr.annoMatrix.fetch("obs", "foo");
+      expect(
+        df
+          .col("foo")
+          .asArray()
+          .every((v) => v === "A")
+      ).toBeTruthy();
+
+      // check that we catch dups
+      expect(() =>
+        xfltr.addObsColumn(
+          { name: "foo", type: "categorical" },
+          Array,
+          "toasty"
+        )
+      ).toThrow("column already exists");
+      expect(() =>
+        xfltr.addObsColumn(
+          { name: "louvain", type: "categorical" },
+          Array,
+          "toasty"
+        )
+      ).toThrow("column already exists");
+    });
+
+    test("dropObsColumn", async () => {
+      let xfltr;
+
+      /* check that we catch attempt to drop readonly dimension */
+      expect(() => crossfilter.dropObsColumn("louvain")).toThrow(
+        "Unknown or readonly obs column"
+      );
+      /* non-existent column */
+      expect(() => crossfilter.dropObsColumn("does-not-exist")).toThrow(
+        "Unknown or readonly obs column"
+      );
+
+      // add a column, then drop it.
+      xfltr = await helperAddTestCol(crossfilter, "foo");
+      xfltr = xfltr.dropObsColumn("foo");
+      expect(
+        xfltr.annoMatrix.schema.annotations.obs.columns.filter(
+          (v) => v.name === "foo"
+        )
+      ).toHaveLength(0);
+      expect(xfltr.annoMatrix.schema.annotations.obsByName.foo).toBeUndefined();
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(xfltr.annoMatrix.fetch("obs", "foo")).rejects.toThrow(
+        "unknown column name"
+      );
+
+      // now same, but ensure we have built an index before doing the drop
+      xfltr = await helperAddTestCol(crossfilter, "bar");
+      xfltr = await xfltr.select("obs", "bar", {
+        mode: "exact",
+        values: "whatever",
+      });
+      xfltr = xfltr.dropObsColumn("bar");
+
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(xfltr.select("obs", "bar", { mode: "all" })).rejects.toThrow(
+        "unknown column name"
+      );
+    });
+
+    test("renameObsColumn", async () => {
+      let xfltr;
+
+      /* catch attempts to rename non-existent or readonly columns */
+      expect(() =>
+        crossfilter.renameObsColumn("does-not-exist", "foo")
+      ).toThrow("Unknown or readonly obs column");
+      expect(() => crossfilter.renameObsColumn("louvain", "foo")).toThrow(
+        "Unknown or readonly obs column"
+      );
+
+      // add a column, then rename it.
+      xfltr = await helperAddTestCol(crossfilter, "foo");
+      xfltr = xfltr.renameObsColumn("foo", "bar");
+      expect(xfltr.annoMatrix.getColumnSchema("obs", "foo")).toBeUndefined();
+      expect(xfltr.annoMatrix.getColumnSchema("obs", "bar")).toMatchObject({
+        name: "bar",
+        type: "categorical",
+      });
+
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(xfltr.annoMatrix.fetch("obs", "foo")).rejects.toThrow(
+        "unknown column name"
+      );
+      const df = await xfltr.annoMatrix.fetch("obs", "bar");
+      expect(df.hasCol("bar")).toBeTruthy();
+
+      // now same, but ensure we have built an index before doing the rename
+      xfltr = await helperAddTestCol(crossfilter, "bar");
+      xfltr = await xfltr.select("obs", "bar", {
+        mode: "exact",
+        values: "whatever",
+      });
+      xfltr = xfltr.renameObsColumn("bar", "xyz");
+
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(xfltr.select("obs", "bar", { mode: "all" })).rejects.toThrow(
+        "unknown column name"
+      );
+      await expect(
+        xfltr.select("obs", "xyz", { mode: "none" })
+      ).resolves.toBeInstanceOf(AnnoMatrixObsCrossfilter);
+    });
+
+    test("addObsAnnoCategory", async () => {
+      let xfltr;
+
+      // catch unknown or readonly columns
+      expect(() => crossfilter.addObsAnnoCategory("louvain", "mumble")).toThrow(
+        "Unknown or readonly obs column"
+      );
+      expect(() =>
+        crossfilter.addObsAnnoCategory("undefined-name", "mumble")
+      ).toThrow("Unknown or readonly obs column");
+
+      // add a column and then add category to it
+      xfltr = await helperAddTestCol(crossfilter, "foo", {
+        name: "foo",
+        type: "categorical",
+        categories: ["unassigned"],
+      });
+      xfltr = xfltr.addObsAnnoCategory("foo", "a-new-label");
+      expect(xfltr.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining(["a-new-label", "unassigned"]),
+      });
+
+      // do it again, dup; should throw
+      expect(() => xfltr.addObsAnnoCategory("foo", "a-new-label")).toThrow(
+        "category already exists"
+      );
+
+      // now same, but ensure we have built an index before doing the operation
+      xfltr = await helperAddTestCol(crossfilter, "bar", {
+        name: "bar",
+        type: "categorical",
+        categories: ["unassigned"],
+      });
+      xfltr = await xfltr.select("obs", "bar", {
+        mode: "exact",
+        values: "something",
+      });
+      xfltr = xfltr.addObsAnnoCategory("bar", "a-new-label");
+      expect(xfltr.annoMatrix.getColumnSchema("obs", "bar")).toMatchObject({
+        name: "bar",
+        type: "categorical",
+        categories: expect.arrayContaining(["a-new-label", "unassigned"]),
+      });
+    });
+
+    test("removeObsAnnoCategory", async () => {
+      let xfltr;
+
+      // catch unknown or readonly categories
+      await expect(() =>
+        crossfilter.removeObsAnnoCategory("louvain", "mumble", "unassigned")
+      ).rejects.toThrow("Unknown or readonly obs column");
+      await expect(() =>
+        crossfilter.removeObsAnnoCategory("undefined-name", "mumble")
+      ).rejects.toThrow("Unknown or readonly obs column");
+
+      xfltr = await helperAddTestCol(crossfilter, "foo", {
+        name: "foo",
+        type: "categorical",
+        categories: ["unassigned", "red", "green", "blue"],
+      });
+      xfltr = await xfltr.select("obs", "foo", { mode: "all" });
+      expect(
+        (await xfltr.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .every((v) => v === "unassigned")
+      ).toBeTruthy();
+      expect(xfltr.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining([
+          "unassigned",
+          "red",
+          "green",
+          "blue",
+        ]),
+      });
+
+      // remove an unused category
+      const xfltr1 = await xfltr.removeObsAnnoCategory("foo", "red", "mumble");
+      expect(
+        (await xfltr1.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .every((v) => v === "unassigned")
+      ).toBeTruthy();
+      expect(xfltr1.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining([
+          "unassigned",
+          "green",
+          "blue",
+          "mumble",
+        ]),
+      });
+
+      // remove a used category
+      const xfltr2 = await xfltr.removeObsAnnoCategory(
+        "foo",
+        "unassigned",
+        "red"
+      );
+      expect(
+        (await xfltr2.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .every((v) => v === "red")
+      ).toBeTruthy();
+      expect(xfltr2.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining(["green", "blue", "red"]),
+      });
+    });
+
+    test("setObsColumnValues", async () => {
+      // catch unknown or readonly categories
+      await expect(() =>
+        crossfilter.setObsColumnValues("louvain", [0, 1], "unassigned")
+      ).rejects.toThrow("Unknown or readonly obs column");
+      await expect(() =>
+        crossfilter.setObsColumnValues("undefined-name", [0], "mumble")
+      ).rejects.toThrow("Unknown or readonly obs column");
+
+      let xfltr = await helperAddTestCol(crossfilter, "foo", {
+        name: "foo",
+        type: "categorical",
+        categories: ["unassigned", "red", "green", "blue"],
+      });
+      xfltr = await xfltr.select("obs", "foo", { mode: "all" });
+
+      // catch unknown row label
+      await expect(() =>
+        xfltr.setObsColumnValues("foo", [-1], "red")
+      ).rejects.toThrow("Unknown row label");
+
+      // set a few rows
+      expect(
+        (await xfltr.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .every((v) => v === "unassigned")
+      ).toBeTruthy();
+      const xfltr1 = await xfltr.setObsColumnValues("foo", [0, 10], "purple");
+      expect(
+        (await xfltr1.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .every(
+            (v, i) =>
+              v === "unassigned" || (v === "purple" && (i === 0 || i === 10))
+          )
+      ).toBeTruthy();
+      expect(xfltr1.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining([
+          "unassigned",
+          "red",
+          "green",
+          "blue",
+          "purple",
+        ]),
+      });
+
+      expect(xfltr1.countSelected()).toEqual(xfltr1.annoMatrix.nObs);
+      const xfltr2 = await xfltr1.select("obs", "foo", {
+        mode: "exact",
+        values: ["purple"],
+      });
+      expect(xfltr2.countSelected()).toEqual(2);
+      expect(xfltr2.allSelectedLabels()).toEqual(Int32Array.from([0, 10]));
+    });
+
+    test("resetObsColumnValues", async () => {
+      // catch unknown or readonly categories
+      await expect(() =>
+        crossfilter.resetObsColumnValues("louvain", "red", "blue")
+      ).rejects.toThrow("Unknown or readonly obs column");
+      await expect(() =>
+        crossfilter.resetObsColumnValues("undefined-name", "red", "blue")
+      ).rejects.toThrow("Unknown or readonly obs column");
+
+      let xfltr = await helperAddTestCol(crossfilter, "foo", {
+        name: "foo",
+        type: "categorical",
+        categories: ["unassigned", "red", "green", "blue"],
+      });
+      xfltr = await xfltr.select("obs", "foo", {
+        mode: "exact",
+        values: "red",
+      });
+
+      // catch unknown category name label
+      await expect(() =>
+        xfltr.resetObsColumnValues("foo", "unknown-label", "red")
+      ).rejects.toThrow("unknown category");
+
+      let xfltr1 = await xfltr.setObsColumnValues("foo", [0, 10], "purple");
+      xfltr1 = await xfltr1.select("obs", "foo", {
+        mode: "exact",
+        values: "purple",
+      });
+      expect(
+        (await xfltr1.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .filter((v) => v === "purple")
+      ).toHaveLength(2);
+
+      xfltr1 = await xfltr1.resetObsColumnValues("foo", "purple", "magenta");
+      expect(
+        (await xfltr1.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .filter((v) => v === "magenta")
+      ).toHaveLength(2);
+      expect(
+        (await xfltr1.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .filter((v) => v === "purple")
+      ).toHaveLength(0);
+      expect(xfltr1.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining([
+          "unassigned",
+          "red",
+          "green",
+          "blue",
+          "purple",
+          "magenta",
+        ]),
+      });
+    });
+  });
+
+  describe("edge cases", () => {
+    test("transition from empty annoMatrix", async () => {
+      // select before fetch needs to work
+      fetch.once(serverMocks.dataframeResponse(["louvain"], [obsLouvain]));
+      const xfltr = await crossfilter.select("obs", "louvain", {
+        mode: "exact",
+        values: "B cells",
+      });
+      expect(fetch.mock.calls).toHaveLength(1);
+      expect(xfltr.obsCrossfilter.hasDimension("obs/louvain")).toBeTruthy();
+      expect(xfltr.obsCrossfilter.all()).toBe(xfltr.annoMatrix._cache.obs);
+      expect(xfltr.countSelected()).toEqual(
+        obsLouvain.reduce(
+          (count, v) => (v === "B cells" ? count + 1 : count),
+          0
+        )
+      );
+    });
+  });
+});

client/__tests__/util/annoMatrix/serverMocks/index.js

@@ -0,0 +1,11 @@
+export const baseDataURL = "https://a.fake.url/api/v0.2";
+
+window.CELLXGENE = {
+  API: {
+    prefix: baseDataURL,
+    version: "v0.2/",
+  },
+};
+
+export { schema } from "./schema";
+export * from "./routes";

client/__tests__/util/annoMatrix/serverMocks/routes.js

@@ -0,0 +1,211 @@
+import { schema } from "./schema";
+import { Dataframe, KeyIndex } from "../../../../src/util/dataframe";
+import { encodeMatrixFBS } from "../../../../src/util/stateManager/matrix";
+
+const indexedSchema = {
+  obsByName: Object.fromEntries(
+    schema.schema.annotations.obs.columns.map((v) => [v.name, v]) ?? []
+  ),
+  varByName: Object.fromEntries(
+    schema.schema.annotations.var.columns.map((v) => [v.name, v]) ?? []
+  ),
+  embByName: Object.fromEntries(
+    schema.schema.layout.obs.map((v) => [v.name, v]) ?? []
+  ),
+};
+
+function makeMockColumn(s, length) {
+  const { type } = s;
+  switch (type) {
+    case "int32":
+      return new Int32Array(length).fill(Math.floor(99 * Math.random()));
+
+    case "string":
+      return new Array(length).fill("test");
+
+    case "float32":
+      return new Float32Array(length).fill(99 * Math.random());
+
+    case "boolean":
+      return new Array(length).fill(false);
+
+    case "categorical":
+      return new Array(length).fill(s.categories[0]);
+
+    default:
+      throw new Error("unkonwn type");
+  }
+}
+
+function getEncodedDataframe(colNames, length, colSchemas) {
+  const colIndex = new KeyIndex(colNames);
+  const columns = colSchemas.map((s) => makeMockColumn(s, length));
+  const df = new Dataframe([length, colNames.length], columns, null, colIndex);
+  const body = encodeMatrixFBS(df);
+  return body;
+}
+
+export function dataframeResponse(colNames, columns) {
+  const colIndex = new KeyIndex(colNames);
+  const df = new Dataframe(
+    [columns[0].length, colNames.length],
+    columns,
+    null,
+    colIndex
+  );
+  const body = encodeMatrixFBS(df);
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return () => Promise.resolve({ body, init: { status: 200, headers } });
+}
+
+function annotationObsResponse(request) {
+  const url = new URL(request.url);
+  const params = Array.from(url.searchParams.entries());
+  const names = params
+    .filter(([k]) => k === "annotation-name")
+    .map(([, v]) => v);
+  if (!names.every((n) => indexedSchema.obsByName[n])) {
+    return Promise.reject(new Error("bad obs annotation name in URL"));
+  }
+  const colSchemas = names.map((n) => indexedSchema.obsByName[n]);
+  const body = getEncodedDataframe(
+    names,
+    schema.schema.dataframe.nObs,
+    colSchemas
+  );
+
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return Promise.resolve({
+    body,
+    init: { status: 200, headers },
+  });
+}
+
+function annotationVarResponse(request) {
+  const url = new URL(request.url);
+  const params = Array.from(url.searchParams.entries());
+  const names = params
+    .filter(([k]) => k === "annotation-name")
+    .map(([, v]) => v);
+  if (!names.every((n) => indexedSchema.varByName[n])) {
+    return Promise.reject(new Error("bad var annotation name in URL"));
+  }
+  const colSchemas = names.map((n) => indexedSchema.varByName[n]);
+  const body = getEncodedDataframe(
+    names,
+    schema.schema.dataframe.nVar,
+    colSchemas
+  );
+
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return Promise.resolve({
+    body,
+    init: { status: 200, headers },
+  });
+}
+
+function layoutObsResponse(request) {
+  const url = new URL(request.url);
+  const params = Array.from(url.searchParams.entries());
+  const names = params.filter(([k]) => k === "layout-name").map(([, v]) => v);
+  if (!names.every((n) => indexedSchema.embByName[n])) {
+    return Promise.reject(new Error("bad layout name in URL"));
+  }
+  const dims = names.map((n) => indexedSchema.embByName[n].dims).flat();
+  const colSchemas = names
+    .map((n) => [indexedSchema.embByName[n], indexedSchema.embByName[n]])
+    .flat();
+  const body = getEncodedDataframe(
+    dims,
+    schema.schema.dataframe.nObs,
+    colSchemas
+  );
+
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return Promise.resolve({
+    body,
+    init: { status: 200, headers },
+  });
+}
+
+function dataVarResponse(request) {
+  const url = new URL(request.url);
+  const params = Array.from(url.searchParams.entries());
+
+  const colNames = params.map((v) => `${v[0]}/${v[1]}`);
+  const colSchemas = colNames.map(() => schema.schema.dataframe);
+  const body = getEncodedDataframe(
+    colNames,
+    schema.schema.dataframe.nObs,
+    colSchemas
+  );
+
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return Promise.resolve({
+    body,
+    init: { status: 200, headers },
+  });
+}
+
+export function responder(request) {
+  const url = new URL(request.url);
+  const { pathname } = url;
+  if (pathname.endsWith("/annotations/obs")) {
+    return annotationObsResponse(request);
+  }
+  if (pathname.endsWith("/annotations/var")) {
+    return annotationVarResponse(request);
+  }
+  if (pathname.endsWith("/layout/obs")) {
+    return layoutObsResponse(request);
+  }
+  if (pathname.endsWith("/data/var")) {
+    return dataVarResponse(request);
+  }
+  return Promise.reject(new Error("bad URL"));
+}
+
+export function withExpected(expectedURL, expectedParams) {
+  /*
+  Do some additional error checking
+  */
+  return (request) => {
+    // if URL is bogus, reject the promise
+    const url = new URL(request.url);
+    if (!url.pathname.endsWith(expectedURL)) {
+      return Promise.reject(new Error("Unexpected URL!"));
+    }
+    const params = Array.from(url.searchParams.entries()).sort(
+      (a, b) => a[0] < b[0]
+    );
+    expectedParams = expectedParams.slice().sort((a, b) => a[0] < b[0]);
+
+    if (
+      params.length !== expectedParams.length ||
+      !params.every(
+        (p, i) => p[0] === expectedParams[i][0] && p[1] === expectedParams[i][1]
+      )
+    ) {
+      return Promise.reject(new Error("unexpected name requested in URL"));
+    }
+
+    return responder(request);
+  };
+}
+
+export function annotationsObs(names) {
+  return withExpected(
+    "/annotations/obs",
+    names.map((name) => ["annotation-name", name])
+  );
+}

client/__tests__/util/annoMatrix/serverMocks/schema.js

@@ -0,0 +1,80 @@
+export const schema = {
+  schema: {
+    annotations: {
+      obs: {
+        columns: [
+          {
+            name: "name_0",
+            type: "string",
+            writable: false,
+          },
+          {
+            name: "n_genes",
+            type: "int32",
+            writable: false,
+          },
+          {
+            name: "percent_mito",
+            type: "float32",
+            writable: false,
+          },
+          {
+            name: "n_counts",
+            type: "float32",
+            writable: false,
+          },
+          {
+            name: "louvain",
+            type: "string",
+            writable: false,
+          },
+        ],
+        index: "name_0",
+      },
+      var: {
+        columns: [
+          {
+            name: "name_0",
+            type: "string",
+            writable: false,
+          },
+          {
+            name: "n_cells",
+            type: "int32",
+            writable: false,
+          },
+        ],
+        index: "name_0",
+      },
+    },
+    dataframe: {
+      nObs: 2638,
+      nVar: 1838,
+      type: "float32",
+    },
+    layout: {
+      obs: [
+        {
+          dims: ["draw_graph_fr_0", "draw_graph_fr_1"],
+          name: "draw_graph_fr",
+          type: "float32",
+        },
+        {
+          dims: ["pca_0", "pca_1"],
+          name: "pca",
+          type: "float32",
+        },
+        {
+          dims: ["tsne_0", "tsne_1"],
+          name: "tsne",
+          type: "float32",
+        },
+        {
+          dims: ["umap_0", "umap_1"],
+          name: "umap",
+          type: "float32",
+        },
+      ],
+    },
+  },
+};

client/__tests__/util/annoMatrix/whereCache.test.js

@@ -0,0 +1,184 @@
+import {
+  _whereCacheGet,
+  _whereCacheCreate,
+  _whereCacheMerge,
+} from "../../../src/annoMatrix/whereCache";
+
+const schema = {};
+
+describe("whereCache", () => {
+  test("whereCacheGet - missing cache values", () => {
+    expect(
+      _whereCacheGet({}, schema, "X", {
+        field: "var",
+        column: "foo",
+        value: "bar",
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet({ X: {} }, schema, "X", {
+        field: "var",
+        column: "foo",
+        value: "bar",
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet({ X: { var: new Map() } }, schema, "X", {
+        field: "var",
+        column: "foo",
+        value: "bar",
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet(
+        { X: { var: new Map([["foo", new Map()]]) } },
+        schema,
+        "X",
+        {
+          field: "var",
+          column: "foo",
+          value: "bar",
+        }
+      )
+    ).toEqual([undefined]);
+  });
+
+  test("whereCacheGet - varied lookups", () => {
+    const whereCache = {
+      X: {
+        var: new Map([
+          [
+            "foo",
+            new Map([
+              ["bar", [0]],
+              ["baz", [1, 2]],
+            ]),
+          ],
+        ]),
+      },
+    };
+
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        field: "var",
+        column: "foo",
+        value: "bar",
+      })
+    ).toEqual([0]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        field: "var",
+        column: "foo",
+        value: "baz",
+      })
+    ).toEqual([1, 2]);
+    expect(_whereCacheGet(whereCache, schema, "Y", {})).toEqual([undefined]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        field: "whoknows",
+        column: "whatever",
+        value: "snork",
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        field: "var",
+        column: "whatever",
+        value: "snork",
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        field: "var",
+        column: "foo",
+        value: "snork",
+      })
+    ).toEqual([undefined]);
+  });
+
+  test("whereCacheCreate", () => {
+    const query = {
+      field: "queryField",
+      column: "queryColumn",
+      value: "queryValue",
+    };
+    const wc = _whereCacheCreate(
+      "field",
+      { field: "queryField", column: "queryColumn", value: "queryValue" },
+      [0, 1, 2]
+    );
+    expect(wc).toBeDefined();
+    expect(wc).toEqual(
+      expect.objectContaining({
+        field: {
+          queryField: expect.any(Map),
+        },
+      })
+    );
+    expect(wc.field.queryField.has("queryColumn")).toEqual(true);
+    expect(wc.field.queryField.get("queryColumn")).toBeInstanceOf(Map);
+    expect(wc.field.queryField.get("queryColumn").has("queryValue")).toEqual(
+      true
+    );
+    expect(_whereCacheGet(wc, schema, "field", query)).toEqual([0, 1, 2]);
+  });
+
+  test("whereCacheMerge", () => {
+    let wc;
+
+    // remember, will mutate dst
+    const src = _whereCacheCreate(
+      "field",
+      { field: "queryField", column: "queryColumn", value: "foo" },
+      ["foo"]
+    );
+
+    const dst1 = _whereCacheCreate(
+      "field",
+      { field: "queryField", column: "queryColumn", value: "bar" },
+      ["dst1"]
+    );
+    wc = _whereCacheMerge(dst1, src);
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        field: "queryField",
+        column: "queryColumn",
+        value: "foo",
+      })
+    ).toEqual(["foo"]);
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        field: "queryField",
+        column: "queryColumn",
+        value: "bar",
+      })
+    ).toEqual(["dst1"]);
+
+    const dst2 = _whereCacheCreate(
+      "field",
+      { field: "queryField", column: "queryColumn", value: "bar" },
+      ["dst2"]
+    );
+    wc = _whereCacheMerge(dst2, dst1, src);
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        field: "queryField",
+        column: "queryColumn",
+        value: "foo",
+      })
+    ).toEqual(["foo"]);
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        field: "queryField",
+        column: "queryColumn",
+        value: "bar",
+      })
+    ).toEqual(["dst1"]);
+
+    wc = _whereCacheMerge({}, src);
+    expect(wc).toEqual(src);
+
+    wc = _whereCacheMerge({ field: { queryField: new Map() } }, src);
+    expect(wc).toEqual(src);
+  });
+});

client/__tests__/util/centroid.test.js

@@ -0,0 +1,75 @@
+import _ from "lodash";
+
+import calcCentroid from "../../src/util/centroid";
+import quantile from "../../src/util/quantile";
+import { matrixFBSToDataframe } from "../../src/util/stateManager/matrix";
+import * as REST from "./stateManager/sampleResponses";
+import { indexEntireSchema } from "../../src/util/stateManager/schemaHelpers";
+import { _normalizeCategoricalSchema } from "../../src/annoMatrix/schema";
+
+describe("centroid", () => {
+  let schema;
+  let obsAnnotations;
+  let obsLayout;
+
+  beforeAll(() => {
+    schema = indexEntireSchema(_.cloneDeep(REST.schema.schema));
+    obsAnnotations = matrixFBSToDataframe(REST.annotationsObs);
+    obsLayout = matrixFBSToDataframe(REST.layoutObs);
+
+    _normalizeCategoricalSchema(
+      schema.annotations.obsByName.field3,
+      obsAnnotations.col("field3")
+    );
+  });
+
+  test("field4 (categorical obsAnnotation)", () => {
+    const centroidResult = calcCentroid(
+      schema,
+      "field4",
+      obsAnnotations,
+      { current: "umap", currentDimNames: ["umap_0", "umap_1"] },
+      obsLayout
+    );
+
+    // Check to see that a centroid has been calculated for every categorical value
+    const keysAsArray = Array.from(centroidResult.keys());
+    expect(keysAsArray).toEqual(
+      expect.arrayContaining([83, true, "foo", 2.222222])
+    );
+
+    // This expected result assumes that all cells belong in all categorical values inside of sample response
+    const expectedResult = [
+      quantile([0.5], obsLayout.col("umap_0").asArray())[0],
+      quantile([0.5], obsLayout.col("umap_1").asArray())[0],
+    ];
+
+    centroidResult.forEach((coordinate) => {
+      expect(coordinate).toEqual(expectedResult);
+    });
+  });
+
+  test("field3 (boolean obsAnnotation)", () => {
+    const centroidResult = calcCentroid(
+      schema,
+      "field3",
+      obsAnnotations,
+      { current: "umap", currentDimNames: ["umap_0", "umap_1"] },
+      obsLayout
+    );
+
+    // Check to see that a centroid has been calculated for every categorical value
+    const keysAsArray = Array.from(centroidResult.keys());
+    expect(keysAsArray).toEqual(expect.arrayContaining([false, true]));
+
+    // This expected result assumes that all cells belong in all categorical values inside of sample response
+    const expectedResult = [
+      quantile([0.5], obsLayout.col("umap_0").asArray())[0],
+      quantile([0.5], obsLayout.col("umap_1").asArray())[0],
+    ];
+
+    centroidResult.forEach((coordinate) => {
+      expect(coordinate).toEqual(expectedResult);
+    });
+  });
+});

client/__tests__/util/dataframe/histogram.test.js

@@ -0,0 +1,103 @@
+import * as Dataframe from "../../../src/util/dataframe";
+
+describe("Dataframe column histogram", () => {
+  test("categorical by categorical", () => {
+    const df = new Dataframe.Dataframe(
+      [3, 3],
+      [["n1", "n2", "n3"], ["c1", "c2", "c3"], new Int32Array([0, 1, 2])],
+      null,
+      new Dataframe.KeyIndex(["name", "cat", "value"])
+    );
+
+    const h1 = df.col("cat").histogram(df.col("name"));
+    expect(h1).toMatchObject(
+      new Map([
+        ["n1", new Map([["c1", 1]])],
+        ["n2", new Map([["c2", 1]])],
+        ["n3", new Map([["c3", 1]])],
+      ])
+    );
+    // memoized?
+    expect(df.col("cat").histogram(df.col("name"))).toMatchObject(h1);
+  });
+
+  test("continuous by categorical", () => {
+    const df = new Dataframe.Dataframe(
+      [3, 3],
+      [["n1", "n2", "n3"], ["c1", "c2", "c3"], new Int32Array([0, 1, 2])],
+      null,
+      new Dataframe.KeyIndex(["name", "cat", "value"])
+    );
+
+    const h1 = df.col("value").histogram(3, [0, 2], df.col("name"));
+    expect(h1).toMatchObject(
+      new Map([
+        ["n1", [1, 0, 0]],
+        ["n2", [0, 1, 0]],
+        ["n3", [0, 0, 1]],
+      ])
+    );
+    // memoized?
+    expect(df.col("value").histogram(3, [0, 2], df.col("name"))).toMatchObject(
+      h1
+    );
+  });
+
+  test("categorical", () => {
+    const df = new Dataframe.Dataframe(
+      [3, 3],
+      [["n1", "n2", "n3"], ["c1", "c2", "c3"], new Int32Array([0, 1, 2])],
+      null,
+      new Dataframe.KeyIndex(["name", "cat", "value"])
+    );
+
+    const h1 = df.col("cat").histogram();
+    expect(h1).toMatchObject(
+      new Map([
+        ["c1", 1],
+        ["c2", 1],
+        ["c3", 1],
+      ])
+    );
+    // memoized?
+    expect(df.col("value").histogram(3, [0, 2])).toMatchObject(h1);
+  });
+
+  test("continuous", () => {
+    const df = new Dataframe.Dataframe(
+      [3, 3],
+      [["n1", "n2", "n3"], ["c1", "c2", "c3"], new Int32Array([0, 1, 2])],
+      null,
+      new Dataframe.KeyIndex(["name", "cat", "value"])
+    );
+
+    const h1 = df.col("value").histogram(3, [0, 2]);
+    expect(h1).toMatchObject([1, 1, 1]);
+    // memoized?
+    expect(df.col("value").histogram(3, [0, 2])).toMatchObject(h1);
+  });
+
+  test("continuous thesholds correct", () => {
+    const vals = [0, 1, 9, 10, 11, 20, 99, 100];
+    const df = new Dataframe.Dataframe(
+      [8, 2],
+      [new Int32Array(vals), new Float32Array(vals)]
+    );
+
+    expect(df.col(0).histogram(5, [0, 100])).toEqual([5, 1, 0, 0, 2]);
+    expect(df.col(1).histogram(5, [0, 100])).toEqual([5, 1, 0, 0, 2]);
+    expect(df.col(0).histogram(2, [0, 10])).toEqual([2, 2]);
+    expect(df.col(0).histogram(10, [0, 100])).toEqual([
+      3,
+      2,
+      1,
+      0,
+      0,
+      0,
+      0,
+      0,
+      0,
+      2,
+    ]);
+  });
+});

client/__tests__/util/dataframe/summarize.test.js

@@ -0,0 +1,281 @@
+import * as Dataframe from "../../../src/util/dataframe";
+
+function float32Conversion(f) {
+  return new Float32Array([f])[0];
+}
+
+describe("Dataframe column summary", () => {
+  test("empty column test", () => {
+    const df = Dataframe.Dataframe.create([0, 1], [[]]);
+    const summary = df.icol(0).summarize();
+    expect(summary).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: [],
+        categoryCounts: new Map(),
+        numCategories: 0,
+      })
+    );
+  });
+
+  test("simple test", () => {
+    const df = new Dataframe.Dataframe(
+      [1, 6],
+      [
+        ["n1"],
+        ["hi"],
+        [true],
+        new Float32Array([39.3]),
+        new Int32Array([99]),
+        [1],
+      ],
+      null,
+      new Dataframe.KeyIndex([
+        "name",
+        "nameString",
+        "nameBoolean",
+        "nameFloat32",
+        "nameInt32",
+        "nameCategorical",
+      ])
+    );
+
+    expect(df.icol(0).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: ["n1"],
+        categoryCounts: new Map([["n1", 1]]),
+        numCategories: 1,
+      })
+    );
+    expect(df.icol(1).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: ["hi"],
+        categoryCounts: new Map([["hi", 1]]),
+        numCategories: 1,
+      })
+    );
+    expect(df.icol(2).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: [true],
+        categoryCounts: new Map([[true, 1]]),
+        numCategories: 1,
+      })
+    );
+    expect(df.icol(3).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: false,
+        min: float32Conversion(39.3),
+        max: float32Conversion(39.3),
+        nan: 0,
+        ninf: 0,
+        pinf: 0,
+      })
+    );
+    expect(df.icol(4).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: false,
+        min: 99,
+        max: 99,
+        nan: 0,
+        ninf: 0,
+        pinf: 0,
+      })
+    );
+    expect(df.icol(5).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: [1],
+        categoryCounts: new Map([[1, 1]]),
+        numCategories: 1,
+      })
+    );
+  });
+
+  test("multi test", () => {
+    const df = new Dataframe.Dataframe(
+      [3, 6],
+      [
+        ["n0", "n1", "n2"],
+        ["hi", "hi", "bye"],
+        [false, true, true],
+        new Float32Array([39.3, 39.3, 0]),
+        new Int32Array([99, 99, 99]),
+        [1, false, "0"],
+      ],
+      null,
+      new Dataframe.KeyIndex([
+        "name",
+        "nameString",
+        "nameBoolean",
+        "nameFloat32",
+        "nameInt32",
+        "nameCategorical",
+      ])
+    );
+
+    expect(df.icol(0).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining(["n0", "n1", "n2"]),
+        categoryCounts: new Map([
+          ["n0", 1],
+          ["n1", 1],
+          ["n2", 1],
+        ]),
+        numCategories: 3,
+      })
+    );
+    expect(df.icol(1).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining(["hi", "bye"]),
+        categoryCounts: new Map([
+          ["hi", 2],
+          ["bye", 1],
+        ]),
+        numCategories: 2,
+      })
+    );
+    expect(df.icol(2).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining([true, false]),
+        categoryCounts: new Map([
+          [true, 2],
+          [false, 1],
+        ]),
+        numCategories: 2,
+      })
+    );
+    expect(df.icol(3).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: false,
+        min: 0,
+        max: float32Conversion(39.3),
+        nan: 0,
+        ninf: 0,
+        pinf: 0,
+      })
+    );
+    expect(df.icol(4).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: false,
+        min: 99,
+        max: 99,
+        nan: 0,
+        ninf: 0,
+        pinf: 0,
+      })
+    );
+    expect(df.icol(5).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining([1, false, "0"]),
+        categoryCounts: new Map([
+          [1, 1],
+          [false, 1],
+          ["0", 1],
+        ]),
+        numCategories: 3,
+      })
+    );
+  });
+
+  test("non-finite numbers", () => {
+    const df = new Dataframe.Dataframe(
+      [4, 6],
+      [
+        ["n0", "n1", "n2", "n2"],
+        ["hi", "hi", "bye", "bye"],
+        [false, true, true, true],
+        new Float32Array([
+          39.3,
+          Number.NEGATIVE_INFINITY,
+          Number.NaN,
+          Number.POSITIVE_INFINITY,
+        ]),
+        new Int32Array([99, 99, 99, 99]),
+        [1, false, "0", "0"],
+      ],
+      null,
+      new Dataframe.KeyIndex([
+        "name",
+        "nameString",
+        "nameBoolean",
+        "nameFloat32",
+        "nameInt32",
+        "nameCategorical",
+      ])
+    );
+
+    expect(df.icol(0).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining(["n0", "n1", "n2"]),
+        categoryCounts: new Map([
+          ["n0", 1],
+          ["n1", 1],
+          ["n2", 2],
+        ]),
+        numCategories: 3,
+      })
+    );
+    expect(df.icol(1).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining(["hi", "bye"]),
+        categoryCounts: new Map([
+          ["hi", 2],
+          ["bye", 1],
+        ]),
+        numCategories: 2,
+      })
+    );
+    expect(df.icol(2).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining([true, false]),
+        categoryCounts: new Map([
+          [true, 2],
+          [false, 1],
+        ]),
+        numCategories: 2,
+      })
+    );
+    expect(df.icol(3).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: false,
+        min: float32Conversion(39.3),
+        max: float32Conversion(39.3),
+        nan: 1,
+        ninf: 1,
+        pinf: 1,
+      })
+    );
+    expect(df.icol(4).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: false,
+        min: 99,
+        max: 99,
+        nan: 0,
+        ninf: 0,
+        pinf: 0,
+      })
+    );
+    expect(df.icol(5).summarize()).toEqual(
+      expect.objectContaining({
+        categorical: true,
+        categories: expect.arrayContaining([1, false, "0"]),
+        categoryCounts: new Map([
+          [1, 1],
+          [false, 1],
+          ["0", 1],
+        ]),
+        numCategories: 3,
+      })
+    );
+  });
+});

client/__tests__/util/nameCreators.test.js

@@ -0,0 +1,68 @@
+/*
+Test cases for nameCreators.js.
+*/
+import {
+  layoutDimensionName,
+  obsAnnoDimensionName,
+  diffexpDimensionName,
+  userDefinedDimensionName,
+  makeContinuousDimensionName,
+} from "../../src/util/nameCreators";
+
+describe("nameCreators", () => {
+  const nameCreators = [
+    layoutDimensionName,
+    obsAnnoDimensionName,
+    diffexpDimensionName,
+    userDefinedDimensionName,
+  ];
+
+  test("check for namespace isolation", () => {
+    const foo = "foo";
+    nameCreators.forEach((fn) => expect(fn(foo)).not.toBe(foo));
+
+    const bar = "bar";
+    nameCreators.forEach((fn) => expect(fn(foo)).not.toBe(fn(bar)));
+
+    nameCreators.forEach((fn) => {
+      const allOtherFn = nameCreators.filter((elmnt) => elmnt !== fn);
+      allOtherFn.forEach((otherFn) => expect(fn(foo)).not.toBe(otherFn(foo)));
+    });
+  });
+
+  test("check for legal keys", () => {
+    /* need namespace creators to return strings only */
+    nameCreators.forEach((fn) => expect(fn("X")).toMatch(/X/));
+  });
+});
+
+describe("makeContinuousDimensionName", () => {
+  test("namespaces", () => {
+    expect(makeContinuousDimensionName({ isObs: true }, "OK")).toMatch(/OK/);
+    expect(makeContinuousDimensionName({ isDiffExp: true }, "OK")).toMatch(
+      /OK/
+    );
+    expect(makeContinuousDimensionName({ isUserDefined: true }, "OK")).toMatch(
+      /OK/
+    );
+  });
+
+  test("error handling", () => {
+    expect(() => makeContinuousDimensionName({}, "oops")).toThrow();
+    expect(() =>
+      makeContinuousDimensionName(
+        { isObs: false, isDiffExp: false, isUserDefined: false },
+        "oops"
+      )
+    ).toThrow();
+    expect(() =>
+      makeContinuousDimensionName({ isObs: true }, "OK")
+    ).not.toThrow();
+    expect(() =>
+      makeContinuousDimensionName({ isDiffExp: true }, "OK")
+    ).not.toThrow();
+    expect(() =>
+      makeContinuousDimensionName({ isUserDefined: true }, "OK")
+    ).not.toThrow();
+  });
+});

client/__tests__/util/promiseLimit.test.js

@@ -0,0 +1,96 @@
+import PromiseLimit from "../../src/util/promiseLimit";
+import { range } from "../../src/util/range";
+
+const delay = (t) => new Promise((resolve) => setTimeout(resolve, t));
+
+describe("PromiseLimit", () => {
+  test("simple evaluation, concurrency 1", async () => {
+    const plimit = new PromiseLimit(1);
+    const result = await Promise.all([
+      plimit.add(() => Promise.resolve(1)),
+      plimit.add(() => Promise.resolve(2)),
+      plimit.add(() => Promise.resolve(3)),
+      plimit.add(() => Promise.resolve(4)),
+    ]);
+    expect(result).toEqual([1, 2, 3, 4]);
+  });
+
+  test("simple evaluation, concurrency > 1", async () => {
+    const plimit = new PromiseLimit(100);
+    const result = await Promise.all([
+      plimit.add(() => Promise.resolve(1)),
+      plimit.add(() => Promise.resolve(2)),
+      plimit.add(() => Promise.resolve(3)),
+      plimit.add(() => Promise.resolve(4)),
+    ]);
+    expect(result).toEqual([1, 2, 3, 4]);
+  });
+
+  test("eval in order of insertion", async () => {
+    const plimit = new PromiseLimit(100);
+
+    const result = await Promise.all([
+      plimit.add(() => Promise.resolve(1)),
+      plimit.add(() => Promise.resolve(2)),
+      plimit.add(() => Promise.resolve(3)),
+      plimit.add(() => Promise.resolve(4)),
+    ]);
+
+    expect(result).toEqual([1, 2, 3, 4]);
+  });
+
+  test("obeys concurrency limit", async () => {
+    const plimit = new PromiseLimit(2);
+    let running = 0;
+    let maxRunning = 0;
+
+    const callback = async () => {
+      running += 1;
+      maxRunning = running > maxRunning ? running : maxRunning;
+      await delay(100);
+      running -= 1;
+    };
+
+    await Promise.all(range(10).map((i) => plimit.add(() => callback(i))));
+
+    expect(maxRunning).toEqual(2);
+  });
+
+  test("rejection", async () => {
+    const plimit = new PromiseLimit(2);
+    const result = await Promise.all([
+      plimit.add(() => Promise.resolve("OK")),
+      // eslint-disable-next-line prefer-promise-reject-errors -- unit test
+      plimit.add(() => Promise.reject("not OK")).catch((e) => e),
+      plimit.add(() => Promise.resolve("OK")),
+      plimit
+        .add(() => {
+          throw new Error("not OK");
+        })
+        .catch((e) => e.message),
+    ]);
+    expect(result).toEqual(["OK", "not OK", "OK", "not OK"]);
+  });
+
+  test("priority queue", async () => {
+    const plimit = new PromiseLimit(1);
+
+    let finishOrder = 0;
+    const callback = () => async () => {
+      await delay(100);
+      const result = finishOrder;
+      finishOrder += 1;
+      return result;
+    };
+
+    const result = await Promise.all([
+      plimit.add(callback()),
+      plimit.priorityAdd(4, callback()),
+      plimit.priorityAdd(0, callback()),
+      plimit.priorityAdd(1, callback()),
+      plimit.priorityAdd(-1, callback()),
+    ]);
+
+    expect(result).toEqual([0, 4, 2, 3, 1]);
+  });
+});

client/__tests__/util/quantile.test.js

@@ -0,0 +1,29 @@
+import quantile from "../../src/util/quantile";
+
+describe("quantile", () => {
+  test("single q", () => {
+    const arr = new Float32Array([9, 3, 5, 6, 0]);
+    expect(quantile([1.0], arr)).toMatchObject([9]);
+    expect(quantile([0.9], arr)).toMatchObject([9]);
+    expect(quantile([0.8], arr)).toMatchObject([9]);
+    expect(quantile([0.7], arr)).toMatchObject([6]);
+    expect(quantile([0.6], arr)).toMatchObject([6]);
+    expect(quantile([0.5], arr)).toMatchObject([5]);
+    expect(quantile([0.4], arr)).toMatchObject([5]);
+    expect(quantile([0.3], arr)).toMatchObject([3]);
+    expect(quantile([0.2], arr)).toMatchObject([3]);
+    expect(quantile([0.1], arr)).toMatchObject([0]);
+    expect(quantile([0], arr)).toMatchObject([0]);
+  });
+
+  test("multi q", () => {
+    const arr = new Float32Array([9, 3, 5, 6, 0]);
+    expect(quantile([0, 0.25, 0.5, 0.75, 1.0], arr)).toMatchObject([
+      0,
+      3,
+      5,
+      6,
+      9,
+    ]);
+  });
+});

client/__tests__/util/range.test.js

@@ -0,0 +1,48 @@
+import { range, rangeFill, linspace } from "../../src/util/range";
+
+describe("range", () => {
+  test("no defaults", () => {
+    expect(range(0, 3, 1)).toMatchObject([0, 1, 2]);
+  });
+
+  test("range(stop)", () => {
+    expect(range(3)).toMatchObject([0, 1, 2]);
+    expect(range(0)).toMatchObject([]);
+    expect(range(1)).toMatchObject([0]);
+  });
+
+  test("range(start,stop)", () => {
+    expect(range(0, 0)).toMatchObject([]);
+    expect(range(0, 2)).toMatchObject([0, 1]);
+    expect(range(4, 8)).toMatchObject([4, 5, 6, 7]);
+  });
+
+  test("range(start, stop, step", () => {
+    expect(range(4, 0, -1)).toMatchObject([4, 3, 2, 1]);
+    expect(range(0, 4, 2)).toMatchObject([0, 2]);
+  });
+});
+
+describe("rangefill", () => {
+  test("rangeFill(arr)", () => {
+    expect(rangeFill(new Int32Array(3))).toMatchObject(
+      new Int32Array([0, 1, 2])
+    );
+  });
+  test("rangeFill(arr, start)", () => {
+    expect(rangeFill(new Int32Array(2), 1)).toMatchObject(
+      new Int32Array([1, 2])
+    );
+  });
+  test("rangeFill(arr, start, step)", () => {
+    expect(rangeFill(new Int32Array(3), 2, -1)).toMatchObject(
+      new Int32Array([2, 1, 0])
+    );
+  });
+});
+
+describe("linspace", () => {
+  test("linspace(arr, start, step)", () => {
+    expect(linspace(0.0, 2.0, 5)).toMatchObject([0.0, 0.5, 1.0, 1.5, 2.0]);
+  });
+});

client/__tests__/util/stateManager/controlsHelpers.test.js

@@ -0,0 +1,38 @@
+/*
+test controls helpers
+*/
+import { subsetAndResetGeneLists } from "../../../src/util/stateManager/controlsHelpers";
+import * as globals from "../../../src/globals";
+
+describe("controls helpers", () => {
+  test("subsetAndResetGeneLists", () => {
+    const geneList = [];
+    const genRandGene = () => Math.random().toString(36).substring(2, 6);
+
+    // build a unique set of genes
+    for (let i = 0; i < 150; i += 1) {
+      let randGene = genRandGene();
+      while (geneList.includes(randGene)) randGene = genRandGene();
+      geneList.push(randGene);
+    }
+    // insert duplicates
+    geneList[0] = "dupl";
+    geneList[20] = "dupl";
+    const state = {
+      userDefinedGenes: geneList.slice(0, 20),
+      diffexpGenes: geneList.slice(20),
+    };
+    const [newUserDefinedGenes, newDiffExpGenes] = subsetAndResetGeneLists(
+      state
+    );
+    const expectedNewUserDefinedGenes = [
+      ...geneList.slice(0, 20),
+      ...geneList.slice(21),
+    ].slice(0, globals.maxGenes);
+    expect(globals.maxUserDefinedGenes).toBeLessThan(globals.maxGenes);
+    expect(geneList.length).toBeGreaterThan(globals.maxGenes);
+    expect(newUserDefinedGenes).toHaveLength(globals.maxGenes);
+    expect(newUserDefinedGenes).toStrictEqual(expectedNewUserDefinedGenes);
+    expect(newDiffExpGenes).toStrictEqual([]);
+  });
+});

client/__tests__/util/stateManager/fbs.test.js

@@ -0,0 +1,34 @@
+/*
+test FBS encode/decode API
+*/
+import { Dataframe, KeyIndex } from "../../../src/util/dataframe";
+import {
+  decodeMatrixFBS,
+  encodeMatrixFBS,
+} from "../../../src/util/stateManager/matrix";
+
+describe("encode/decode", () => {
+  test("round trip", () => {
+    const columns = [
+      ["red", "green", "blue"],
+      new Int32Array(3).fill(0),
+      new Uint32Array(3).fill(1),
+      new Float32Array(3).fill(2),
+    ];
+
+    const dfNoColIdx = new Dataframe([3, 4], columns);
+    const dfA = decodeMatrixFBS(encodeMatrixFBS(dfNoColIdx));
+    expect([dfA.nRows, dfA.nCols]).toEqual(dfNoColIdx.dims);
+    expect(dfA.colIdx).toBeNull();
+    expect(dfA.rowIdx).toBeNull();
+    expect(dfA.columns).toEqual(columns);
+
+    const colIndex = new KeyIndex(["a", "b", "c", "d"]);
+    const dfWithColIdx = new Dataframe([3, 4], columns, null, colIndex);
+    const dfB = decodeMatrixFBS(encodeMatrixFBS(dfWithColIdx));
+    expect([dfB.nRows, dfB.nCols]).toEqual(dfWithColIdx.dims);
+    expect(dfB.colIdx).toEqual(colIndex.labels());
+    expect(dfB.rowIdx).toBeNull();
+    expect(dfB.columns).toEqual(columns);
+  });
+});

client/__tests__/util/stateManager/sampleResponses.js

@@ -0,0 +1,202 @@
+import _ from "lodash";
+import { flatbuffers } from "flatbuffers";
+import { NetEncoding } from "../../../src/util/stateManager/matrix_generated";
+
+/*
+test data mocking REST 0.2 API responses.  Used in several tests.
+*/
+const nObs = 10;
+const nVar = 32;
+const field4Categories = [83, true, "foo", 2.222222];
+const fieldDCategories = [99, false, "mumble", 3.1415];
+
+const aConfigResponse = {
+  config: {
+    features: [
+      { method: "POST", path: "/cluster/", available: false },
+      { method: "POST", path: "/layout/", available: false },
+      { method: "POST", path: "/diffexp/", available: false },
+      { method: "POST", path: "/saveLocal/", available: false },
+    ],
+    displayNames: {
+      engine: "the little engine that could",
+      dataset: "all your zeros are mine",
+    },
+  },
+};
+
+const aSchemaResponse = {
+  schema: {
+    dataframe: {
+      nObs,
+      nVar,
+      type: "float32",
+    },
+    annotations: {
+      obs: {
+        index: "name",
+        columns: [
+          { name: "name", type: "string" },
+          { name: "field1", type: "int32" },
+          { name: "field2", type: "float32" },
+          { name: "field3", type: "boolean" },
+          {
+            name: "field4",
+            type: "categorical",
+            categories: field4Categories,
+          },
+        ],
+      },
+      var: {
+        index: "name",
+        columns: [
+          { name: "name", type: "string" },
+          { name: "fieldA", type: "int32" },
+          { name: "fieldB", type: "float32" },
+          { name: "fieldC", type: "boolean" },
+          {
+            name: "fieldD",
+            type: "categorical",
+            categories: fieldDCategories,
+          },
+        ],
+      },
+    },
+    layout: {
+      obs: [{ name: "umap", type: "float32", dims: ["umap_0", "umap_1"] }],
+      var: [],
+    },
+  },
+};
+
+const anAnnotationsObsJSONResponse = {
+  names: ["name", "field1", "field2", "field3", "field4"],
+  data: _()
+    .range(nObs)
+    .map((idx) => [
+      idx,
+      `obs${idx}`,
+      2 * idx,
+      idx + 0.0133,
+      // eslint-disable-next-line no-bitwise -- idx & 1 to check for odd numbers
+      !!(idx & 1),
+      field4Categories[idx % field4Categories.length],
+    ])
+    .value(),
+};
+
+const anAnnotationsVarJSONResponse = {
+  names: ["fieldA", "fieldB", "fieldC", "fieldD", "name"],
+  data: _()
+    .range(nVar)
+    .map((idx) => [
+      idx,
+      10 * idx,
+      idx + 2.90143,
+      // eslint-disable-next-line no-bitwise -- idx & 1 to check for odd numbers
+      !!(idx & 1),
+      fieldDCategories[idx % fieldDCategories.length],
+      `var${idx}`,
+    ])
+    .value(),
+};
+
+function encodeTypedArray(builder, uType, uData) {
+  const uTypeName = NetEncoding.TypedArray[uType];
+  const ArrayType = NetEncoding[uTypeName];
+  const dv = ArrayType.createDataVector(builder, uData);
+  builder.startObject(1);
+  builder.addFieldOffset(0, dv, 0);
+  return builder.endObject();
+}
+
+function encodeMatrix(columns, colIndex = undefined) {
+  /*
+  IMPORTANT: this is not a general purpose encoder.  in particular,
+  it doesn't correctly handle all column index types, nor does it
+  handle all column typedarray types.
+
+  encodeMatrixFBS in matrix.py is more general.  This is used only
+  as a testing santity check (alt implementation).
+  */
+  const utf8Encoder = new TextEncoder("utf-8");
+  const builder = new flatbuffers.Builder(1024);
+  const cols = _.map(columns, (carr) => {
+    let uType;
+    let tarr;
+    if (_.every(carr, _.isNumber)) {
+      uType = NetEncoding.TypedArray.Float32Array;
+      tarr = encodeTypedArray(builder, uType, new Float32Array(carr));
+    } else {
+      uType = NetEncoding.TypedArray.JSONEncodedArray;
+      const json = JSON.stringify(carr);
+      const jsonUTF8 = utf8Encoder.encode(json);
+      tarr = encodeTypedArray(builder, uType, jsonUTF8);
+    }
+    NetEncoding.Column.startColumn(builder);
+    NetEncoding.Column.addUType(builder, uType);
+    NetEncoding.Column.addU(builder, tarr);
+    return NetEncoding.Column.endColumn(builder);
+  });
+
+  const encColumns = NetEncoding.Matrix.createColumnsVector(builder, cols);
+
+  let encColIndex;
+  if (colIndex) {
+    encColIndex = encodeTypedArray(
+      builder,
+      NetEncoding.TypedArray.JSONEncodedArray,
+      utf8Encoder.encode(JSON.stringify(colIndex))
+    );
+  }
+
+  NetEncoding.Matrix.startMatrix(builder);
+  NetEncoding.Matrix.addNRows(builder, columns[0].length);
+  NetEncoding.Matrix.addNCols(builder, columns.length);
+  NetEncoding.Matrix.addColumns(builder, encColumns);
+  if (colIndex) {
+    NetEncoding.Matrix.addColIndexType(
+      builder,
+      NetEncoding.TypedArray.JSONEncodedArray
+    );
+    NetEncoding.Matrix.addColIndex(builder, encColIndex);
+  }
+  const root = NetEncoding.Matrix.endMatrix(builder);
+  builder.finish(root);
+  return builder.asUint8Array();
+}
+
+const anAnnotationsObsFBSResponse = (() => {
+  const columns = _.zip(...anAnnotationsObsJSONResponse.data).slice(1);
+  return encodeMatrix(columns, anAnnotationsObsJSONResponse.names);
+})();
+
+const anAnnotationsVarFBSResponse = (() => {
+  const columns = _.zip(...anAnnotationsVarJSONResponse.data).slice(1);
+  return encodeMatrix(columns, anAnnotationsVarJSONResponse.names);
+})();
+
+const aLayoutFBSResponse = (() => {
+  const coords = [
+    new Float32Array(nObs).fill(Math.random()),
+    new Float32Array(nObs).fill(Math.random()),
+  ];
+  return encodeMatrix(coords, ["umap_0", "umap_1"]);
+})();
+
+const aDataObsResponse = {
+  var: [2, 4, 29],
+  obs: _()
+    .range(nObs)
+    .map((idx) => [idx, Math.random(), Math.random(), Math.random()])
+    .value(),
+};
+
+export {
+  aLayoutFBSResponse as layoutObs,
+  aDataObsResponse as dataObs,
+  anAnnotationsVarFBSResponse as annotationsVar,
+  anAnnotationsObsFBSResponse as annotationsObs,
+  aSchemaResponse as schema,
+  aConfigResponse as config,
+};

client/__tests__/util/typedCrossfilter/bitArray.test.js

@@ -0,0 +1,206 @@
+// jshint esversion: 6
+
+import BitArray from "../../../src/util/typedCrossfilter/bitArray";
+
+const defaultTestLength = 8;
+
+describe("default select state", () => {
+  test("newly created Bitarray should be deselected", () => {
+    const ba = new BitArray(defaultTestLength);
+    expect(ba).toBeDefined();
+
+    for (let i = 0; i < defaultTestLength; i += 1) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+
+    const dim = ba.allocDimension();
+    expect(dim).toBeDefined();
+    for (let i = 0; i < defaultTestLength; i += 1) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+
+    ba.freeDimension(dim);
+    for (let i = 0; i < defaultTestLength; i += 1) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+  });
+});
+
+describe("select and deselect", () => {
+  test("selectAll and deselectAll", () => {
+    const ba = new BitArray(defaultTestLength);
+    expect(ba).toBeDefined();
+    const dim1 = ba.allocDimension();
+    expect(dim1).toBeDefined();
+    ba.selectAll(dim1);
+
+    for (let i = 0; i < defaultTestLength; i += 1) {
+      expect(ba.isSelected(i)).toEqual(true);
+    }
+
+    const dim2 = ba.allocDimension();
+    expect(dim2).toBeDefined();
+    for (let i = 0; i < defaultTestLength; i += 1) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+
+    ba.selectAll(dim2);
+    for (let i = 0; i < defaultTestLength; i += 1) {
+      expect(ba.isSelected(i)).toEqual(true);
+    }
+
+    ba.deselectAll(dim1);
+    for (let i = 0; i < defaultTestLength; i += 1) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+
+    ba.deselectAll(dim2);
+    for (let i = 0; i < defaultTestLength; i += 1) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+
+    ba.selectAll(dim1);
+    ba.selectAll(dim2);
+    for (let i = 0; i < defaultTestLength; i += 1) {
+      expect(ba.isSelected(i)).toEqual(true);
+    }
+
+    ba.freeDimension(dim1);
+    for (let i = 0; i < defaultTestLength; i += 1) {
+      expect(ba.isSelected(i)).toEqual(true);
+    }
+
+    ba.freeDimension(dim2);
+    for (let i = 0; i < defaultTestLength; i += 1) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+  });
+
+  test("selectOne and deselectOne", () => {
+    const ba = new BitArray(defaultTestLength);
+    expect(ba).toBeDefined();
+    const dim = ba.allocDimension();
+    expect(dim).toBeDefined();
+
+    ba.selectOne(dim, 0);
+    expect(ba.isSelected(0)).toEqual(true);
+    for (let i = 1; i < defaultTestLength; i += 1) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+
+    ba.deselectOne(dim, 0);
+    for (let i = 0; i < defaultTestLength; i += 1) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+
+    ba.selectOne(dim, 1);
+    expect(ba.isSelected(1)).toEqual(true);
+    expect(ba.isSelected(0)).toEqual(false);
+    for (let i = 2; i < defaultTestLength; i += 1) {
+      expect(ba.isSelected(i)).toEqual(false);
+    }
+
+    ba.selectAll(dim);
+    ba.deselectOne(dim, defaultTestLength - 1);
+    expect(ba.isSelected(defaultTestLength - 1)).toEqual(false);
+    for (let i = 0; i < defaultTestLength - 1; i += 1) {
+      expect(ba.isSelected(i)).toEqual(true);
+    }
+  });
+});
+
+describe("selectionCount", () => {
+  test("simple", () => {
+    const ba = new BitArray(defaultTestLength);
+    expect(ba).toBeDefined();
+    const dim1 = ba.allocDimension();
+    expect(dim1).toBeDefined();
+    const dim2 = ba.allocDimension();
+    expect(dim2).toBeDefined();
+
+    expect(ba.selectionCount()).toEqual(0);
+    ba.selectAll(dim1);
+    expect(ba.selectionCount()).toEqual(0);
+    ba.selectAll(dim2);
+    expect(ba.selectionCount()).toEqual(defaultTestLength);
+
+    for (let i = 0; i < defaultTestLength; i += 1) {
+      ba.deselectOne(dim1, i);
+      expect(ba.selectionCount()).toEqual(defaultTestLength - i - 1);
+      expect(ba.selectionCount()).toEqual(ba.countAllOnes());
+    }
+
+    ba.freeDimension(dim1);
+    ba.freeDimension(dim2);
+  });
+});
+
+describe("fillBySelection", () => {
+  test("sets values correctly", () => {
+    const ba = new BitArray(defaultTestLength);
+    expect(ba).toBeDefined();
+    const dim = ba.allocDimension();
+    expect(dim).toBeDefined();
+
+    const arr = new Int32Array(defaultTestLength);
+    arr.fill(0);
+    const truth = new Int32Array(defaultTestLength);
+    truth.fill(0);
+
+    // initial state should be deselected
+    ba.fillBySelection(arr, 1, 0);
+    expect(arr).toEqual(expect.not.arrayContaining([1]));
+
+    // selectAll
+    ba.selectAll(dim);
+    ba.fillBySelection(arr, 1, 0);
+    expect(arr).toEqual(expect.not.arrayContaining([0]));
+
+    // deselectOne
+    ba.deselectOne(dim, 3);
+    ba.fillBySelection(arr, 1, 0);
+    truth.fill(1);
+    truth[3] = 0;
+    expect(arr).toEqual(truth);
+
+    // deselectAll
+    ba.deselectAll(dim);
+    ba.fillBySelection(arr, 1, 0);
+    truth.fill(0);
+    expect(arr).toEqual(truth);
+
+    // selectOne
+    ba.selectOne(dim, 5);
+    ba.fillBySelection(arr, 6, 1);
+    truth.fill(1);
+    truth[5] = 6;
+    expect(arr).toEqual(truth);
+
+    // should be deselected after dimension disposal
+    ba.freeDimension(dim);
+    ba.fillBySelection(arr, 3, 9);
+    truth.fill(9);
+    expect(arr).toEqual(truth);
+  });
+});
+
+describe("wide bitarray", () => {
+  test.each([9, 30, 31, 32, 33, 54, 63, 64, 65, 127, 128, 129])(
+    "more than %d dimensions",
+    (ndim) => {
+      /* ensure we move across the uint boundary correctly */
+
+      const ba = new BitArray(defaultTestLength);
+      expect(ba).toBeDefined();
+
+      for (let i = 0; i < ndim; i += 1) {
+        expect(ba.allocDimension()).toEqual(i);
+      }
+      ba.freeDimension(0);
+      expect(ba.allocDimension()).toEqual(0);
+
+      ba.freeDimension(ndim - 1);
+      expect(ba.allocDimension()).toEqual(ndim - 1);
+    }
+  );
+});

client/__tests__/util/typedCrossfilter/crossfilter.test.js

@@ -0,0 +1,498 @@
+import _ from "lodash";
+
+import Crossfilter from "../../../src/util/typedCrossfilter";
+
+const someData = [
+  {
+    date: "2011-11-14T16:17:54Z",
+    quantity: 2,
+    total: 190,
+    tip: 100,
+    type: "tab",
+    productIDs: ["001"],
+    coords: [0, 0],
+    nonFinite: 0.0,
+  },
+  {
+    date: "2011-11-14T16:20:19Z",
+    quantity: 2,
+    total: 190,
+    tip: 100,
+    type: "tab",
+    productIDs: ["001", "005"],
+    coords: [0.4, 0.4],
+    nonFinite: Number.NaN,
+  },
+  {
+    date: "2011-11-14T16:28:54Z",
+    quantity: 1,
+    total: 300,
+    tip: 200,
+    type: "visa",
+    productIDs: ["004", "005"],
+    coords: [0.3, 0.1],
+    nonFinite: Number.POSITIVE_INFINITY,
+  },
+  {
+    date: "2011-11-14T16:30:43Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["001", "002"],
+    coords: [0.392, 0.1],
+    nonFinite: Number.NEGATIVE_INFINITY,
+  },
+  {
+    date: "2011-11-14T16:48:46Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["005"],
+    coords: [0.7, 0.0482],
+    nonFinite: 1.0,
+  },
+  {
+    date: "2011-11-14T16:53:41Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["001", "004", "005"],
+    coords: [0.9999, 1.0],
+    nonFinite: Number.NaN,
+  },
+  {
+    date: "2011-11-14T16:54:06Z",
+    quantity: 1,
+    total: 100,
+    tip: 0,
+    type: "cash",
+    productIDs: ["001", "002", "003", "004", "005"],
+    coords: [0.384, 0.6938],
+    nonFinite: 99.0,
+  },
+  {
+    date: "2011-11-14T16:58:03Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["001"],
+    coords: [0.4822, 0.482],
+    nonFinite: Number.NaN,
+  },
+  {
+    date: "2011-11-14T17:07:21Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["004", "005"],
+    coords: [0.2234, 0],
+    nonFinite: Number.NaN,
+  },
+  {
+    date: "2011-11-14T17:22:59Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["001", "002", "004", "005"],
+    coords: [0.382, 0.38485],
+    nonFinite: -1,
+  },
+  {
+    date: "2011-11-14T17:25:45Z",
+    quantity: 2,
+    total: 200,
+    tip: 0,
+    type: "cash",
+    productIDs: ["002"],
+    coords: [0.998, 0.8472],
+    nonFinite: 0.0,
+  },
+  {
+    date: "2011-11-14T17:29:52Z",
+    quantity: 1,
+    total: 200,
+    tip: 100,
+    type: "visa",
+    productIDs: ["004"],
+    coords: [0.8273, 0.3384],
+    nonFinite: 0.0,
+  },
+];
+
+let payments = null;
+beforeEach(() => {
+  payments = new Crossfilter(someData);
+});
+
+describe("ImmutableTypedCrossfilter", () => {
+  test("create crossfilter", () => {
+    expect(payments).toBeDefined();
+    expect(payments.size()).toEqual(someData.length);
+    expect(payments.all()).toEqual(someData);
+
+    const p = payments
+      .addDimension("quantity", "scalar", (i, d) => d[i].quantity, Int32Array)
+      .select("quantity", { mode: "all" });
+    expect(p).toBeDefined();
+    expect(p.all()).toEqual(someData);
+    expect(p.size()).toEqual(someData.length);
+    expect(p.isElementSelected(0)).toBeTruthy();
+    expect(p.countSelected()).toEqual(someData.length);
+    expect(p.allSelected()).toEqual(someData);
+  });
+
+  test("immutability", () => {
+    /*
+    the following should return a new crossfilter:
+    - addDimension()
+    - delDimension()
+    - select
+    */
+    const p2 = payments.addDimension(
+      "quantity",
+      "scalar",
+      (i, data) => data[i].quantity,
+      Int32Array
+    );
+
+    expect(payments).not.toBe(p2);
+    const p3 = p2.select("quantity", { mode: "all" });
+    expect(p3).not.toBe(p2);
+
+    const p4 = p3.delDimension("quantity");
+    expect(p4).not.toBe(p3);
+
+    const p5 = p2.renameDimension("quantity", "Quantity");
+    expect(p5).not.toBe(p2);
+  });
+
+  test("select all and none", () => {
+    let p = payments
+      .addDimension("quantity", "scalar", (i, d) => d[i].quantity, Int32Array)
+      .addDimension("tip", "scalar", (i, d) => d[i].tip, Float32Array)
+      .addDimension("total", "scalar", (i, d) => d[i].total, Float32Array)
+      .addDimension("type", "enum", (i, d) => d[i].type);
+    expect(p).toBeDefined();
+
+    /* expect all records to be selected - default init state */
+    expect(p.allSelected()).toEqual(someData);
+    expect(p.countSelected()).toEqual(someData.length);
+    expect(p.allSelectedMask()).toEqual(
+      new Uint8Array(someData.length).fill(1)
+    );
+    expect(p.fillByIsSelected(new Uint8Array(someData.length), 99, 0)).toEqual(
+      new Uint8Array(someData.length).fill(99)
+    );
+    for (let i = 0; i < someData.length; i += 1) {
+      expect(p.isElementSelected(i)).toBeTruthy();
+    }
+
+    /* expect a selectAll on one dimension to change nothing */
+    p = p.select("tip", { mode: "all" });
+    expect(p.allSelected()).toEqual(someData);
+
+    /* ditto */
+    p = p.select("quantity", { mode: "all" });
+    expect(p.allSelected()).toEqual(someData);
+
+    /* select none on one dimension */
+    p = p.select("type", { mode: "none" });
+    expect(p.allSelected()).toEqual([]);
+    expect(p.countSelected()).toEqual(0);
+    expect(p.allSelectedMask()).toEqual(
+      new Uint8Array(someData.length).fill(0)
+    );
+    expect(p.fillByIsSelected(new Uint8Array(someData.length), 99, 0)).toEqual(
+      new Uint8Array(someData.length).fill(0)
+    );
+    for (let i = 0; i < someData.length; i += 1) {
+      expect(p.isElementSelected(i)).toBeFalsy();
+    }
+
+    p = p.select("quantity", { mode: "none" });
+    expect(p.allSelected()).toEqual([]);
+
+    // invert the first none; should have no effect because type is
+    // still not filtered.
+    p = p.select("quantity", { mode: "all" });
+    expect(p.allSelected()).toEqual([]);
+
+    /* select all of type; should select all records */
+    p = p.select("type", { mode: "all" });
+    expect(p.allSelected()).toEqual(someData);
+  });
+
+  describe("scalar dimension", () => {
+    let p;
+    beforeEach(() => {
+      p = payments
+        .addDimension("quantity", "scalar", (i, d) => d[i].quantity, Int32Array)
+        .addDimension("tip", "scalar", (i, d) => d[i].tip, Float32Array)
+        .select("tip", { mode: "all" });
+    });
+
+    /*
+    select modes:  all, none, exact, range
+    */
+    test("all", () => {
+      expect(p.select("quantity", { mode: "all" }).countSelected()).toEqual(
+        someData.length
+      );
+    });
+    test("none", () => {
+      expect(p.select("quantity", { mode: "none" }).countSelected()).toEqual(0);
+    });
+    test.each([[[]], [[2]], [[2, 1]], [[9, 82]], [[0, 1]]])("exact: %p", (v) =>
+      expect(
+        p.select("quantity", { mode: "exact", values: v }).countSelected()
+      ).toEqual(_.filter(someData, (d) => v.includes(d.quantity)).length)
+    );
+    test("single value exact", () => {
+      expect(
+        p.select("quantity", { mode: "exact", values: 2 }).countSelected()
+      ).toEqual(_.filter(someData, (d) => d.quantity === 2).length);
+    });
+    test.each([
+      [0, 1],
+      [1, 2],
+      [0, 99],
+      [99, 100000],
+    ])("range %p", (lo, hi) =>
+      expect(
+        p.select("quantity", { mode: "range", lo, hi }).countSelected()
+      ).toEqual(
+        _.filter(someData, (d) => d.quantity >= lo && d.quantity < hi).length
+      )
+    );
+    test("bad mode", () => {
+      expect(() => p.select("type", { mode: "bad mode" })).toThrow(Error);
+    });
+  });
+
+  describe("enum dimension", () => {
+    let p;
+    beforeEach(() => {
+      p = payments.addDimension("type", "enum", (i, d) => d[i].type);
+    });
+
+    test("all", () => {
+      expect(p.select("type", { mode: "all" }).countSelected()).toEqual(
+        someData.length
+      );
+    });
+    test("none", () => {
+      expect(p.select("type", { mode: "none" }).countSelected()).toEqual(0);
+    });
+    test.each([
+      [[]],
+      [["tab"]],
+      [["visa"]],
+      [["visa", "tab"]],
+      [["cash", "tab", "visa"]],
+    ])("exact: %p", (v) =>
+      expect(
+        p.select("type", { mode: "exact", values: v }).countSelected()
+      ).toEqual(_.filter(someData, (d) => v.includes(d.type)).length)
+    );
+    test("single value exact", () => {
+      expect(
+        p.select("type", { mode: "exact", values: "tab" }).countSelected()
+      ).toEqual(_.filter(someData, (d) => d.type === "tab").length);
+    });
+    test("range", () => {
+      expect(() => p.select("type", { mode: "range", lo: 0, hi: 9 })).toThrow(
+        Error
+      );
+    });
+    test("bad mode", () => {
+      expect(() => p.select("type", { mode: "bad mode" })).toThrow(Error);
+    });
+  });
+
+  describe("spatial dimension", () => {
+    let p;
+    beforeEach(() => {
+      const X = someData.map((r) => r.coords[0]);
+      const Y = someData.map((r) => r.coords[1]);
+      p = payments.addDimension("coords", "spatial", X, Y);
+    });
+
+    test("all", () => {
+      expect(p.select("coords", { mode: "all" }).countSelected()).toEqual(
+        someData.length
+      );
+    });
+    test("none", () => {
+      expect(p.select("coords", { mode: "none" }).countSelected()).toEqual(0);
+    });
+    test.each([
+      [0, 0, 1, 1],
+      [0, 0, 0.5, 0.5],
+      [0.5, 0.5, 1, 1],
+    ])("within-rect %d %d %d %d", (minX, minY, maxX, maxY) => {
+      expect(
+        p
+          .select("coords", { mode: "within-rect", minX, minY, maxX, maxY })
+          .allSelected()
+      ).toEqual(
+        _.filter(someData, (d) => {
+          const [x, y] = d.coords;
+          return minX <= x && x < maxX && minY <= y && y < maxY;
+        })
+      );
+    });
+
+    test.each([
+      [
+        [
+          [0, 0],
+          [0, 1],
+          [1, 1],
+          [1, 0],
+        ],
+        [
+          true,
+          true,
+          true,
+          true,
+          true,
+          false,
+          true,
+          true,
+          true,
+          true,
+          true,
+          true,
+        ],
+      ],
+      [
+        [
+          [0, 0],
+          [0, 0.5],
+          [0.5, 0.5],
+          [0.5, 0],
+        ],
+        [
+          true,
+          true,
+          true,
+          true,
+          false,
+          false,
+          false,
+          true,
+          true,
+          true,
+          false,
+          false,
+        ],
+      ],
+    ])("within-polygon %p", (polygon, expected) => {
+      expect(
+        p.select("coords", { mode: "within-polygon", polygon }).allSelected()
+      ).toEqual(
+        _.zip(someData, expected)
+          .filter((x) => x[1])
+          .map((x) => x[0])
+      );
+    });
+  });
+
+  describe("non-finite scalars", () => {
+    let p;
+    beforeEach(() => {
+      p = payments
+        .addDimension("quantity", "scalar", (i, d) => d[i].quantity, Int32Array)
+        .addDimension(
+          "nonFinite",
+          "scalar",
+          (i, d) => d[i].nonFinite,
+          Float32Array
+        )
+        .select("quantity", { mode: "all" });
+    });
+
+    test("all or none", () => {
+      expect(p.select("nonFinite", { mode: "all" }).countSelected()).toEqual(
+        someData.length
+      );
+      expect(p.select("nonFinite", { mode: "none" }).countSelected()).toEqual(
+        0
+      );
+    });
+
+    test("exact", () => {
+      expect(
+        p.select("nonFinite", { mode: "exact", values: [0] }).countSelected()
+      ).toEqual(3);
+      expect(
+        p.select("nonFinite", { mode: "exact", values: [1] }).countSelected()
+      ).toEqual(1);
+      expect(
+        p
+          .select("nonFinite", {
+            mode: "exact",
+            values: [Number.POSITIVE_INFINITY],
+          })
+          .countSelected()
+      ).toEqual(1);
+      expect(
+        p
+          .select("nonFinite", {
+            mode: "exact",
+            values: [Number.NEGATIVE_INFINITY],
+          })
+          .countSelected()
+      ).toEqual(1);
+      expect(
+        p
+          .select("nonFinite", { mode: "exact", values: [Number.NaN] })
+          .countSelected()
+      ).toEqual(4);
+      expect(
+        p
+          .select("nonFinite", {
+            mode: "exact",
+            values: [Number.POSITIVE_INFINITY, 0, 1, 99],
+          })
+          .countSelected()
+      ).toEqual(6);
+    });
+
+    test("range", () => {
+      expect(
+        p
+          .select("nonFinite", {
+            mode: "range",
+            lo: 0,
+            hi: Number.POSITIVE_INFINITY,
+          })
+          .countSelected()
+      ).toEqual(5);
+      expect(
+        p
+          .select("nonFinite", {
+            mode: "range",
+            lo: 0,
+            hi: Number.NaN,
+          })
+          .countSelected()
+      ).toEqual(6);
+      expect(
+        p
+          .select("nonFinite", {
+            mode: "range",
+            lo: Number.NEGATIVE_INFINITY,
+            hi: Number.POSITIVE_INFINITY,
+          })
+          .countSelected()
+      ).toEqual(7);
+    });
+  });
+});

client/__tests__/util/typedCrossfilter/positiveInterval.test.js

@@ -0,0 +1,235 @@
+// jshint esversion: 6
+
+// const PositiveIntervals = require("../../src/util/typedCrossfilter/positiveIntervals");
+import PositiveIntervals from "../../../src/util/typedCrossfilter/positiveIntervals";
+
+describe("canonicalize", () => {
+  test("empty", () => {
+    expect(PositiveIntervals.canonicalize([])).toEqual([]);
+  });
+
+  test("simple, already correct", () => {
+    expect(PositiveIntervals.canonicalize([[0, 1]])).toEqual([[0, 1]]);
+    expect(
+      PositiveIntervals.canonicalize([
+        [0, 1],
+        [2, 3],
+      ])
+    ).toEqual([
+      [0, 1],
+      [2, 3],
+    ]);
+  });
+
+  test("non-canonical, need to be canonicalized", () => {
+    expect(
+      PositiveIntervals.canonicalize([
+        [0, 1],
+        [1, 2],
+      ])
+    ).toEqual([[0, 2]]);
+    expect(
+      PositiveIntervals.canonicalize([
+        [1, 2],
+        [2, 3],
+      ])
+    ).toEqual([[1, 3]]);
+  });
+});
+
+describe("union", () => {
+  test("empty range", () => {
+    expect(PositiveIntervals.union([], [])).toEqual([]);
+    expect(PositiveIntervals.union([], [[1, 2]])).toEqual([[1, 2]]);
+    expect(
+      PositiveIntervals.union(
+        [],
+        [
+          [1, 2],
+          [3, 4],
+        ]
+      )
+    ).toEqual([
+      [1, 2],
+      [3, 4],
+    ]);
+    expect(PositiveIntervals.union([[3, 4]], [])).toEqual([[3, 4]]);
+    expect(
+      PositiveIntervals.union(
+        [
+          [1, 2],
+          [3, 4],
+        ],
+        []
+      )
+    ).toEqual([
+      [1, 2],
+      [3, 4],
+    ]);
+    expect(PositiveIntervals.union([[3, 3]], [])).toEqual([[3, 3]]);
+    expect(PositiveIntervals.union([], [[3, 3]])).toEqual([[3, 3]]);
+  });
+
+  test("simple ranges", () => {
+    expect(PositiveIntervals.union([[1, 2]], [[2, 3]])).toEqual([[1, 3]]);
+    expect(PositiveIntervals.union([[2, 3]], [[1, 2]])).toEqual([[1, 3]]);
+    expect(PositiveIntervals.union([[1, 2]], [[3, 4]])).toEqual([
+      [1, 2],
+      [3, 4],
+    ]);
+    expect(
+      PositiveIntervals.union(
+        [
+          [1, 2],
+          [3, 4],
+        ],
+        [
+          [6, 7],
+          [19, 40],
+        ]
+      )
+    ).toEqual([
+      [1, 2],
+      [3, 4],
+      [6, 7],
+      [19, 40],
+    ]);
+    expect(
+      PositiveIntervals.union(
+        [[1, 4]],
+        [
+          [1, 1],
+          [3, 4],
+        ]
+      )
+    ).toEqual([[1, 4]]);
+    expect(PositiveIntervals.union([[3, 3]], [[4, 4]])).toEqual([
+      [3, 3],
+      [4, 4],
+    ]);
+  });
+});
+
+describe("intersection", () => {
+  test("empty range", () => {
+    expect(PositiveIntervals.intersection([], [])).toEqual([]);
+    expect(PositiveIntervals.intersection([], [[1, 2]])).toEqual([]);
+    expect(PositiveIntervals.intersection([[1, 2]], [])).toEqual([]);
+  });
+
+  test("simple", () => {
+    expect(PositiveIntervals.intersection([[1, 2]], [[2, 3]])).toEqual([]);
+    expect(PositiveIntervals.intersection([[2, 3]], [[1, 2]])).toEqual([]);
+    expect(PositiveIntervals.intersection([[1, 10]], [[1, 10]])).toEqual([
+      [1, 10],
+    ]);
+    expect(PositiveIntervals.intersection([[1, 10]], [[2, 8]])).toEqual([
+      [2, 8],
+    ]);
+    expect(PositiveIntervals.intersection([[2, 8]], [[1, 10]])).toEqual([
+      [2, 8],
+    ]);
+    expect(PositiveIntervals.intersection([[1, 10]], [[2, 12]])).toEqual([
+      [2, 10],
+    ]);
+    expect(PositiveIntervals.intersection([[2, 12]], [[1, 10]])).toEqual([
+      [2, 10],
+    ]);
+    expect(PositiveIntervals.intersection([[1, 10]], [[1, 8]])).toEqual([
+      [1, 8],
+    ]);
+    expect(PositiveIntervals.intersection([[1, 8]], [[1, 10]])).toEqual([
+      [1, 8],
+    ]);
+    expect(
+      PositiveIntervals.intersection(
+        [[1, 10]],
+        [
+          [1, 2],
+          [6, 9],
+        ]
+      )
+    ).toEqual([
+      [1, 2],
+      [6, 9],
+    ]);
+    expect(
+      PositiveIntervals.intersection([[0, 2638]], [[1363, 2638]])
+    ).toEqual([[1363, 2638]]);
+    expect(PositiveIntervals.intersection([[1, 2]], [[1, 2]])).toEqual([
+      [1, 2],
+    ]);
+  });
+});
+
+describe("difference", () => {
+  test("empty", () => {
+    expect(PositiveIntervals.difference([], [])).toEqual([]);
+    expect(PositiveIntervals.difference([], [[1, 10]])).toEqual([]);
+    expect(PositiveIntervals.difference([[1, 10]], [])).toEqual([[1, 10]]);
+  });
+
+  test("simple", () => {
+    expect(
+      PositiveIntervals.difference(
+        [
+          [1, 2],
+          [3, 4],
+        ],
+        []
+      )
+    ).toEqual([
+      [1, 2],
+      [3, 4],
+    ]);
+    expect(
+      PositiveIntervals.difference(
+        [
+          [1, 2],
+          [3, 10],
+        ],
+        [[5, 10]]
+      )
+    ).toEqual([
+      [1, 2],
+      [3, 5],
+    ]);
+    expect(
+      PositiveIntervals.difference(
+        [
+          [1, 2],
+          [3, 10],
+        ],
+        [[0, 5]]
+      )
+    ).toEqual([[5, 10]]);
+    expect(
+      PositiveIntervals.difference(
+        [[0, 2638]],
+        [
+          [0, 1363],
+          [2055, 2638],
+        ]
+      )
+    ).toEqual([[1363, 2055]]);
+    expect(
+      PositiveIntervals.difference(
+        [
+          [0, 1363],
+          [2055, 2638],
+        ],
+        [[0, 2638]]
+      )
+    ).toEqual([]);
+    expect(PositiveIntervals.difference([[0, 10]], [[0, 1]])).toEqual([
+      [1, 10],
+    ]);
+    expect(PositiveIntervals.difference([[0, 10]], [[1, 2]])).toEqual([
+      [0, 1],
+      [2, 10],
+    ]);
+    expect(PositiveIntervals.difference([[0, 10]], [[9, 10]])).toEqual([
+      [0, 9],
+    ]);
+  });
+});

client/__tests__/util/typedCrossfilter/sort.test.js

@@ -0,0 +1,254 @@
+import {
+  sortArray,
+  sortIndex,
+  lowerBound,
+} from "../../../src/util/typedCrossfilter/sort";
+
+/*
+Sort tests should keep in mind that there are separate code
+paths for:
+  - small vs. large arrays (insertionsort only)
+  - float-only typed arrays vs. other array types (non-finite handling)
+  - indexed vs. direct sort
+*/
+
+const pInf = Number.POSITIVE_INFINITY;
+const nInf = Number.NEGATIVE_INFINITY;
+
+function fillRange(arr, start = 0) {
+  const larr = arr;
+  for (let i = 0, len = larr.length; i < len; i += 1) {
+    larr[i] = i + start;
+  }
+  return larr;
+}
+
+function fillRand(arr) {
+  for (let i = 0, len = arr.length; i < len; i += 1) {
+    arr[i] = Math.random();
+  }
+  return arr;
+}
+
+describe("sortArray", () => {
+  describe("JS vals", () => {
+    [
+      [true, false],
+      ["a", "b", "0", "1"],
+      [0, "a", true, null, undefined, 3.1415],
+      fillRand(new Array(1000)),
+      ["a", NaN, null, pInf],
+    ].map((val, idx) =>
+      test(`JS vals ${idx}`, () => {
+        expect(sortArray(val)).toMatchObject(val.sort());
+      })
+    );
+  });
+
+  describe("finite numbers", () => {
+    [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map((Type) =>
+      test(Type.name, () => {
+        expect(sortArray(Type.from([6, 5, 4, 3, 2, 1, 0]))).toMatchObject(
+          Type.from([0, 1, 2, 3, 4, 5, 6])
+        );
+
+        expect(sortArray(Type.from([6, 5, 4, 3, 2, 1]))).toMatchObject(
+          Type.from([1, 2, 3, 4, 5, 6])
+        );
+
+        const source = fillRand(new Type(1000));
+        expect(sortArray(Type.from(source))).toMatchObject(
+          Type.from(source).sort()
+        );
+      })
+    );
+  });
+
+  describe("non-finite numbers", () => {
+    test("infinity", () => {
+      expect(sortArray(new Float32Array([pInf, nInf, 0, 1, 2]))).toMatchObject(
+        new Float32Array([nInf, 0, 1, 2, pInf])
+      );
+      expect(
+        sortArray(new Float32Array([pInf, nInf, pInf, nInf]))
+      ).toMatchObject(new Float32Array([nInf, nInf, pInf, pInf]));
+      expect(
+        sortArray(new Float32Array([pInf, nInf, pInf, nInf, pInf]))
+      ).toMatchObject(new Float32Array([nInf, nInf, pInf, pInf, pInf]));
+      expect(
+        sortArray(
+          new Float32Array(100).fill(Infinity, 0, 50).fill(-Infinity, 50, 100)
+        )
+      ).toMatchObject(
+        new Float32Array(100).fill(-Infinity, 0, 50).fill(Infinity, 50, 100)
+      );
+    });
+    test("NaN", () => {
+      expect(sortArray(new Float64Array([NaN, 2, 1, 0]))).toMatchObject(
+        new Float64Array([0, 1, 2, NaN])
+      );
+      expect(sortArray(new Float32Array([NaN, 2, 1, 0]))).toMatchObject(
+        new Float32Array([0, 1, 2, NaN])
+      );
+      expect(sortArray(new Float32Array([NaN, 2, NaN, 1, 0]))).toMatchObject(
+        new Float32Array([0, 1, 2, NaN, NaN])
+      );
+      expect(sortArray(new Float32Array([NaN, 2, 1, NaN, 0]))).toMatchObject(
+        new Float32Array([0, 1, 2, NaN, NaN])
+      );
+      expect(
+        sortArray(fillRange(new Float32Array(100)).fill(NaN, 0, 10))
+      ).toMatchObject(fillRange(new Float32Array(100), 10).fill(NaN, 90, 100));
+    });
+
+    test("mixed numbers", () => {
+      expect(
+        sortArray(new Float32Array([NaN, pInf, nInf, NaN, NaN]))
+      ).toMatchObject(new Float32Array([nInf, pInf, NaN, NaN, NaN]));
+      expect(
+        sortArray(new Float32Array([NaN, pInf, nInf, NaN, 1, NaN, 2]))
+      ).toMatchObject(new Float32Array([nInf, 1, 2, pInf, NaN, NaN, NaN]));
+      expect(
+        sortArray(new Float32Array([NaN, pInf, nInf, 0, 1, NaN, 2]))
+      ).toMatchObject(new Float32Array([nInf, 0, 1, 2, pInf, NaN, NaN]));
+      expect(
+        sortArray(
+          fillRange(new Float32Array(100))
+            .fill(NaN, 0, 10)
+            .fill(Infinity, 10, 20)
+        )
+      ).toMatchObject(
+        fillRange(new Float32Array(100), 20)
+          .fill(Infinity, 80, 90)
+          .fill(NaN, 90, 100)
+      );
+    });
+  });
+});
+
+describe("sortIndex", () => {
+  describe("finite numbers", () => {
+    [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map((Type) =>
+      test(Type.name, () => {
+        const source1 = Type.from([6, 5, 4, 3, 2, 1, 0]);
+        const index1 = fillRange(new Uint32Array(source1.length));
+        expect(sortIndex(index1, source1)).toMatchObject(
+          index1.sort((a, b) => source1[a] - source1[b])
+        );
+
+        const source2 = Type.from([6, 5, 4, 3, 2, 1]);
+        const index2 = fillRange(new Uint32Array(source2.length));
+        expect(sortIndex(index2, source2)).toMatchObject(
+          index2.sort((a, b) => source1[a] - source1[b])
+        );
+
+        const source3 = fillRand(new Type(1000));
+        const index3 = fillRange(new Uint32Array(source3.length));
+        expect(sortIndex(index3, source3)).toMatchObject(
+          index3.sort((a, b) => source1[a] - source1[b])
+        );
+      })
+    );
+  });
+
+  describe("non-finite numbers", () => {
+    test("mixed numbers", () => {
+      const source1 = new Float32Array([NaN, pInf, nInf, NaN, 1, NaN, 2]);
+      const index1 = fillRange(new Uint32Array(source1.length));
+      expect(sortIndex(index1, source1)).toMatchObject(
+        new Uint32Array([2, 4, 6, 1, 0, 3, 5])
+      );
+
+      const source2 = new Float32Array([NaN, pInf, nInf, 0, 1, NaN, 2]);
+      const index2 = fillRange(new Uint32Array(source2.length));
+      expect(sortIndex(index2, source2)).toMatchObject(
+        new Uint32Array([2, 3, 4, 6, 1, 0, 5])
+      );
+    });
+  });
+});
+
+describe("lowerBound", () => {
+  test("non-float path", () => {
+    expect(lowerBound([], 0, 0, 0)).toEqual(0);
+
+    expect(lowerBound([0, 1, 2, 3], -1, 0, 4)).toEqual(0);
+    expect(lowerBound([0, 1, 2, 3], 0, 0, 4)).toEqual(0);
+    expect(lowerBound([0, 1, 2, 3], 1, 0, 4)).toEqual(1);
+    expect(lowerBound([0, 1, 2, 3], 3, 0, 4)).toEqual(3);
+    expect(lowerBound([0, 1, 2, 3], 4, 0, 4)).toEqual(4);
+    expect(lowerBound([0, 1, 2, 3], 4, 0, 3)).toEqual(3);
+
+    expect(lowerBound([0, 1, 2, 3, 4], -1, 0, 5)).toEqual(0);
+    expect(lowerBound([0, 1, 2, 3, 4], 0, 0, 5)).toEqual(0);
+    expect(lowerBound([0, 1, 2, 3, 4], 2, 0, 5)).toEqual(2);
+    expect(lowerBound([0, 1, 2, 3, 4], 4, 0, 5)).toEqual(4);
+    expect(lowerBound([0, 1, 2, 3, 4], 5, 0, 5)).toEqual(5);
+
+    expect(lowerBound([0, 2, 4, 6, 8], 5, 0, 5)).toEqual(3);
+    expect(lowerBound([0, 2, 2, 2, 8], 5, 0, 5)).toEqual(4);
+
+    expect(lowerBound([0, 1, 2, 3, 4, 5, 6, 7, 8], 3, 2, 4)).toEqual(3);
+    expect(lowerBound([0, 1, 2, 3, 4, 5, 6, 7, 8], 99, 2, 4)).toEqual(4);
+  });
+
+  test("float path, finites", () => {
+    expect(lowerBound(new Float32Array([0, 1, 2, 3]), 1, 0, 4)).toEqual(1);
+
+    expect(lowerBound(new Float32Array([]), 0, 0, 0)).toEqual(0);
+
+    expect(lowerBound(new Float32Array([0, 1, 2, 3]), -1, 0, 4)).toEqual(0);
+    expect(lowerBound(new Float32Array([0, 1, 2, 3]), 0, 0, 4)).toEqual(0);
+    expect(lowerBound(new Float32Array([0, 1, 2, 3]), 1, 0, 4)).toEqual(1);
+    expect(lowerBound(new Float32Array([0, 1, 2, 3]), 3, 0, 4)).toEqual(3);
+    expect(lowerBound(new Float32Array([0, 1, 2, 3]), 4, 0, 4)).toEqual(4);
+    expect(lowerBound(new Float32Array([0, 1, 2, 3]), 4, 0, 3)).toEqual(3);
+
+    expect(lowerBound(new Float32Array([0, 1, 2, 3, 4]), -1, 0, 5)).toEqual(0);
+    expect(lowerBound(new Float32Array([0, 1, 2, 3, 4]), 0, 0, 5)).toEqual(0);
+    expect(lowerBound(new Float32Array([0, 1, 2, 3, 4]), 2, 0, 5)).toEqual(2);
+    expect(lowerBound(new Float32Array([0, 1, 2, 3, 4]), 4, 0, 5)).toEqual(4);
+    expect(lowerBound(new Float32Array([0, 1, 2, 3, 4]), 5, 0, 5)).toEqual(5);
+
+    expect(lowerBound(new Float32Array([0, 2, 4, 6, 8]), 5, 0, 5)).toEqual(3);
+    expect(lowerBound(new Float32Array([0, 2, 2, 2, 8]), 5, 0, 5)).toEqual(4);
+
+    expect(
+      lowerBound(new Float32Array([0, 1, 2, 3, 4, 5, 6, 7, 8]), 3, 2, 4)
+    ).toEqual(3);
+    expect(
+      lowerBound(new Float32Array([0, 1, 2, 3, 4, 5, 6, 7, 8]), 99, 2, 4)
+    ).toEqual(4);
+  });
+
+  test("float path, non-finite", () => {
+    expect(
+      lowerBound(
+        new Float32Array([-Infinity, 0, 1, Infinity, NaN]),
+        -Infinity,
+        0,
+        5
+      )
+    ).toEqual(0);
+    expect(
+      lowerBound(new Float32Array([-Infinity, 0, 1, Infinity, NaN]), 0, 0, 5)
+    ).toEqual(1);
+    expect(
+      lowerBound(new Float32Array([-Infinity, 0, 1, Infinity, NaN]), 1, 0, 5)
+    ).toEqual(2);
+    expect(
+      lowerBound(new Float32Array([-Infinity, 0, 1, Infinity, NaN]), 2, 0, 5)
+    ).toEqual(3);
+    expect(
+      lowerBound(
+        new Float32Array([-Infinity, 0, 1, Infinity, NaN]),
+        Infinity,
+        0,
+        5
+      )
+    ).toEqual(3);
+    expect(
+      lowerBound(new Float32Array([-Infinity, 0, 1, Infinity, NaN]), NaN, 0, 5)
+    ).toEqual(4);
+  });
+});

client/__tests__/util/typedCrossfilter/util.test.js

@@ -0,0 +1,112 @@
+import {
+  sliceByIndex,
+  makeSortIndex,
+} from "../../../src/util/typedCrossfilter/util";
+import { rangeFill as fillRange } from "../../../src/util/range";
+
+describe("fillRange", () => {
+  test("Array", () => {
+    expect(fillRange(new Array(6))).toMatchObject([0, 1, 2, 3, 4, 5]);
+    expect(fillRange(new Array(4), 1)).toMatchObject([1, 2, 3, 4]);
+    expect(fillRange([])).toMatchObject([]);
+  });
+
+  test("Uint32Array", () => {
+    expect(fillRange(new Uint32Array(6))).toMatchObject(
+      new Uint32Array([0, 1, 2, 3, 4, 5])
+    );
+    expect(fillRange(new Array(4), 1)).toMatchObject(
+      new Uint32Array([1, 2, 3, 4])
+    );
+    expect(fillRange([])).toMatchObject(new Uint32Array([]));
+  });
+});
+
+describe("sliceByIndex", () => {
+  test("Array", () => {
+    expect(sliceByIndex([0, 1, 2, 3, 4], [0, 1, 2])).toMatchObject([0, 1, 2]);
+    expect(sliceByIndex([0, 1, 2, 3, 4], [2, 1, 0])).toMatchObject([2, 1, 0]);
+    expect(sliceByIndex([0, 1, 2, 3, 4], [])).toMatchObject([]);
+    expect(sliceByIndex([], [])).toMatchObject([]);
+  });
+
+  test("Uint32Array", () => {
+    expect(
+      sliceByIndex([0, 1, 2, 3, 4], new Uint32Array([0, 1, 2]))
+    ).toMatchObject([0, 1, 2]);
+    expect(
+      sliceByIndex([0, 1, 2, 3, 4], new Uint32Array([2, 1, 0]))
+    ).toMatchObject([2, 1, 0]);
+    expect(sliceByIndex([0, 1, 2, 3, 4], new Uint32Array([]))).toMatchObject(
+      []
+    );
+    expect(sliceByIndex([], new Uint32Array([]))).toMatchObject([]);
+
+    expect(
+      sliceByIndex(new Uint32Array([0, 1, 2, 3, 4]), new Uint32Array([0, 1, 2]))
+    ).toMatchObject(new Uint32Array([0, 1, 2]));
+    expect(
+      sliceByIndex(new Uint32Array([0, 1, 2, 3, 4]), new Uint32Array([2, 1, 0]))
+    ).toMatchObject(new Uint32Array([2, 1, 0]));
+    expect(
+      sliceByIndex(
+        new Uint32Array([0, 1, 2, 3, 4]),
+        new Uint32Array(new Uint32Array([]))
+      )
+    ).toMatchObject(new Uint32Array([]));
+    expect(
+      sliceByIndex(new Uint32Array([]), new Uint32Array([]))
+    ).toMatchObject(new Uint32Array([]));
+  });
+
+  test("Float32Array", () => {
+    expect(
+      sliceByIndex(new Float32Array([0, 1, 2, 3, 4]), [0, 1, 2])
+    ).toMatchObject(new Float32Array([0, 1, 2]));
+    expect(
+      sliceByIndex(new Float32Array([0, 1, 2, 3, 4]), [2, 1, 0])
+    ).toMatchObject(new Float32Array([2, 1, 0]));
+    expect(sliceByIndex(new Float32Array([0, 1, 2, 3, 4]), [])).toMatchObject(
+      new Float32Array([])
+    );
+    expect(sliceByIndex(new Float32Array([]), [])).toMatchObject(
+      new Float32Array([])
+    );
+  });
+});
+
+describe("makeSortIndex", () => {
+  test("Array", () => {
+    expect(makeSortIndex([3, 2, 1, 0])).toMatchObject(
+      new Uint32Array([3, 2, 1, 0])
+    );
+    expect(makeSortIndex([3, 2, 1, 0, 4])).toMatchObject(
+      new Uint32Array([3, 2, 1, 0, 4])
+    );
+    expect(makeSortIndex([])).toMatchObject(new Uint32Array([]));
+  });
+
+  test("Float32Array", () => {
+    expect(makeSortIndex(new Float32Array([3, 2, 1, 0]))).toMatchObject(
+      new Uint32Array([3, 2, 1, 0])
+    );
+    expect(makeSortIndex(new Float32Array([3, 2, 1, 0, 4]))).toMatchObject(
+      new Uint32Array([3, 2, 1, 0, 4])
+    );
+    expect(makeSortIndex(new Float32Array([]))).toMatchObject(
+      new Uint32Array([])
+    );
+  });
+
+  test("Int32Array", () => {
+    expect(makeSortIndex(new Int32Array([3, 2, 1, 0]))).toMatchObject(
+      new Uint32Array([3, 2, 1, 0])
+    );
+    expect(makeSortIndex(new Int32Array([3, 2, 1, 0, 4]))).toMatchObject(
+      new Uint32Array([3, 2, 1, 0, 4])
+    );
+    expect(makeSortIndex(new Int32Array([]))).toMatchObject(
+      new Uint32Array([])
+    );
+  });
+});

client/configuration/babel/babel.dev.js

@@ -0,0 +1,23 @@
+module.exports = {
+  babelrc: false,
+  cacheDirectory: true,
+  presets: [
+    [
+      "@babel/preset-env",
+      {
+        useBuiltIns: "entry",
+        corejs: 3,
+        modules: false,
+      },
+    ],
+    "@babel/preset-react",
+  ],
+  plugins: [
+    "@babel/plugin-proposal-function-bind",
+    ["@babel/plugin-proposal-decorators", { legacy: true }],
+    ["@babel/plugin-proposal-class-properties", { loose: true }],
+    "@babel/plugin-proposal-export-namespace-from",
+    "@babel/plugin-proposal-optional-chaining",
+    "@babel/plugin-proposal-nullish-coalescing-operator",
+  ],
+};

client/configuration/babel/babel.prod.js

@@ -0,0 +1,24 @@
+module.exports = {
+  babelrc: false,
+  presets: [
+    [
+      "@babel/preset-env",
+      {
+        useBuiltIns: "entry",
+        corejs: 3,
+        modules: false,
+      },
+    ],
+    "@babel/preset-react",
+  ],
+  plugins: [
+    "@babel/plugin-proposal-function-bind",
+    ["@babel/plugin-proposal-decorators", { legacy: true }],
+    ["@babel/plugin-proposal-class-properties", { loose: true }],
+    "@babel/plugin-proposal-export-namespace-from",
+    "@babel/plugin-transform-react-constant-elements",
+    "@babel/plugin-transform-runtime",
+    "@babel/plugin-proposal-optional-chaining",
+    "@babel/plugin-proposal-nullish-coalescing-operator",
+  ],
+};

client/configuration/eslint/eslint.js

@@ -0,0 +1,86 @@
+module.exports = {
+  root: true,
+  parser: "babel-eslint",
+  extends: [
+    "airbnb",
+    "plugin:eslint-comments/recommended",
+    "plugin:compat/recommended",
+    "plugin:prettier/recommended",
+    "prettier/react",
+  ],
+  settings: {
+    polyfills: [
+      "TextDecoder",
+      "TextEncoder",
+      "fetch",
+      "Request",
+      "Response",
+      "Headers",
+      "AbortController",
+    ],
+  },
+  env: { browser: true, commonjs: true, es6: true },
+  globals: {
+    expect: true,
+    jest: true,
+    jestPuppeteer: true,
+    it: true,
+    page: true,
+    browser: true,
+    context: true,
+    beforeEach: true,
+  },
+  parserOptions: {
+    ecmaVersion: 2017,
+    sourceType: "module",
+    ecmaFeatures: {
+      jsx: true,
+      generators: true,
+    },
+  },
+  rules: {
+    "eslint-comments/require-description": ["error"],
+    "no-magic-numbers": "off",
+    "no-nested-ternary": "off",
+    "func-style": "off",
+    "arrow-parens": "off",
+    "no-use-before-define": "off",
+    "react/jsx-filename-extension": "off",
+    "comma-dangle": "off",
+    "no-underscore-dangle": "off",
+    "implicit-arrow-linebreak": "off",
+    "no-console": "off",
+    "spaced-comment": ["error", "always", { exceptions: ["*"] }],
+    "no-param-reassign": "off",
+    "object-curly-newline": ["error", { consistent: true }],
+    "react/prop-types": [0],
+    "space-before-function-paren": "off",
+    "function-paren-newline": "off",
+    "prefer-destructuring": ["error", { object: true, array: false }],
+    "import/prefer-default-export": "off",
+    "no-restricted-syntax": [
+      "error",
+      "ForInStatement",
+      "LabeledStatement",
+      "WithStatement",
+    ],
+  },
+  overrides: [
+    {
+      files: ["**/*.test.js"],
+      env: {
+        jest: true, // now **/*.test.js files' env has both es6 *and* jest
+      },
+      // Can't extend in overrides: https://github.com/eslint/eslint/issues/8813
+      // "extends": ["plugin:jest/recommended"]
+      plugins: ["jest"],
+      rules: {
+        "jest/no-disabled-tests": "warn",
+        "jest/no-focused-tests": "error",
+        "jest/no-identical-title": "error",
+        "jest/prefer-to-have-length": "warn",
+        "jest/valid-expect": "error",
+      },
+    },
+  ],
+};

client/configuration/lint-staged/lint-staged.config.js

@@ -0,0 +1,3 @@
+module.exports = {
+  "*.js": "eslint --fix",
+};

client/configuration/webpack/cspHashPlugin.js

@@ -0,0 +1,53 @@
+/* eslint-disable import/no-extraneous-dependencies -- this file is a devDependency*/
+const cheerio = require("cheerio");
+const crypto = require("crypto");
+const HtmlWebpackPlugin = require("html-webpack-plugin");
+
+const digest = (str) => {
+  const hash = crypto.createHash("sha256").update(str, "utf8").digest("base64");
+  return `sha256-${hash}`;
+};
+class CspHashPlugin {
+  constructor(opts) {
+    this.opts = { ...opts };
+  }
+
+  apply(compiler) {
+    compiler.hooks.compilation.tap("CspHashPlugin", (compilation) => {
+      HtmlWebpackPlugin.getHooks(compilation).beforeEmit.tapAsync(
+        "CspHashPlugin",
+        (data, cb) => {
+          const { filename } = this.opts;
+
+          const $ = cheerio.load(data.html, { decodeEntities: false });
+
+          if (filename) {
+            const results = {};
+            results["script-hashes"] = $("script:not([src]):not([no-csp-hash])")
+              .map((i, elmt) => digest($(elmt).html()))
+              .get();
+
+            const json = JSON.stringify(results);
+            compilation.assets[filename] = {
+              source: () => json,
+              size: () => json.length,
+            };
+          }
+
+          // Remove no-csp-hash attributes. Cheerio does not parse Jinja templates
+          // correctly, so we brute force this with a regular expression.
+          data.html = data.html.replace(
+            /(<script .*)no-csp-hash(.*>)/,
+            (match, p1, p2) => [p1, p2].join("")
+          );
+
+          // Tell webpack to move on
+          cb(null, data);
+        }
+      );
+    });
+  }
+}
+
+module.exports = CspHashPlugin;
+/* eslint-enable import/no-extraneous-dependencies -- enable*/

client/configuration/webpack/obsoleteHTMLTemplate.html

@@ -0,0 +1,82 @@
+<script>
+  /** Reflect ANY changes to this script in the script hash in `server/eb/app.py` **/
+  var root = document.getElementById("root");
+  root.remove();
+  var portals = document.getElementsByClassName("bp3-portal");
+  for (var i = 0; i < portals.length; i += 1) {
+    portals[i].remove();
+  }
+</script>
+<div
+  style="
+    display: flex;
+    flex-direction: column;
+    width: 100vw;
+    height: 100vh;
+    text-align: center;
+    justify-content: center;
+    align-items: center;
+    background: #8080801a;
+    font-family: 'Roboto Condensed, sans serif';
+  "
+>
+  <img
+    src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/cellxgene-logo.png"
+    style="width: 320px;"
+  />
+  <div
+    style="
+      margin-top: 16px;
+      background: white;
+      width: 40vw;
+      border-radius: 4px;
+      padding: 24px 64px;
+      -webkit-box-shadow: 0px 0px 3px 2px rgba(0, 0, 0, 0.38);
+      -moz-box-shadow: 0px 0px 3px 2px rgba(0, 0, 0, 0.38);
+      box-shadow: 0px 0px 3px 2px rgba(0, 0, 0, 0.38);
+      max-width: 550px;
+    "
+  >
+    <div style="margin-bottom: 0; font-weight: bolder; font-size: 1.2em;">
+      Unsupported Browser
+    </div>
+    <div style="margin-top: 0;">
+      cellxgene is currently supported on the following browsers
+    </div>
+    <div
+      style="display: flex; justify-content: space-around; margin-top: 16px;"
+    >
+      <a
+        href="https://www.google.com/chrome/?hl=en%22"
+        aria-label="Download Google Chrome"
+      >
+        <img
+          src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/chrome.png"
+          style="width: 80px; height: 80px;"
+        />
+        <div>Chrome &gt; 60</div>
+      </a>
+      <a href="https://www.apple.com/safari/" aria-label="Download Safari">
+        <img
+          src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/safari.png"
+          style="width: 80px; height: 80px;"
+        />
+        <div>Safari ≥ 10.1</div>
+      </a>
+      <a href="https://www.mozilla.com/firefox/" aria-label="Download Firefox">
+        <img
+          src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/firefox.png"
+          style="width: 80px; height: 80px;"
+        />
+        <div>Firefox ≥ 60</div>
+      </a>
+      <a href="//www.microsoft.com/edge" aria-label="Download Edge">
+        <img
+          src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/edge.png"
+          style="width: 80px; height: 80px;"
+        />
+        <div>Edge ≥ 15</div>
+      </a>
+    </div>
+  </div>
+</div>

client/configuration/webpack/webpack.config.dev.js

@@ -0,0 +1,76 @@
+const path = require("path");
+const webpack = require("webpack");
+const HtmlWebpackPlugin = require("html-webpack-plugin");
+const FaviconsWebpackPlugin = require("favicons-webpack-plugin");
+const ScriptExtHtmlWebpackPlugin = require("script-ext-html-webpack-plugin");
+const MiniCssExtractPlugin = require("mini-css-extract-plugin");
+
+const { merge } = require("webpack-merge");
+
+const sharedConfig = require("./webpack.config.shared");
+const babelOptions = require("../babel/babel.dev");
+
+const fonts = path.resolve("src/fonts");
+const nodeModules = path.resolve("node_modules");
+
+const devConfig = {
+  mode: "development",
+  devtool: "eval",
+  output: {
+    pathinfo: true,
+    filename: "static/js/bundle.js",
+  },
+  module: {
+    rules: [
+      {
+        test: /\.jsx?$/,
+        loader: "babel-loader",
+        options: babelOptions,
+      },
+      {
+        test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i,
+        loader: "file-loader",
+        include: [nodeModules, fonts],
+        query: { name: "static/assets/[name].[ext]" },
+      },
+    ],
+  },
+  plugins: [
+    new HtmlWebpackPlugin({
+      inject: true,
+      template: path.resolve("index.html"),
+    }),
+    new FaviconsWebpackPlugin({
+      logo: "./favicon.png",
+      prefix: "static/img/",
+      favicons: {
+        icons: {
+          android: false,
+          appleIcon: false,
+          appleStartup: false,
+          coast: false,
+          firefox: false,
+          windows: false,
+          yandex: false,
+        },
+      },
+    }),
+    new MiniCssExtractPlugin({
+      filename: "static/[name].css",
+    }),
+    new webpack.NoEmitOnErrorsPlugin(),
+    new webpack.DefinePlugin({
+      __REACT_DEVTOOLS_GLOBAL_HOOK__: "({ isDisabled: true })",
+    }),
+    new webpack.DefinePlugin({
+      "process.env.CXG_SERVER_PORT": JSON.stringify(
+        process.env.CXG_SERVER_PORT
+      ),
+    }),
+    new ScriptExtHtmlWebpackPlugin({
+      async: "obsolete",
+    }),
+  ],
+};
+
+module.exports = merge(sharedConfig, devConfig);

client/configuration/webpack/webpack.config.prod.js

@@ -0,0 +1,92 @@
+const path = require("path");
+const HtmlWebpackPlugin = require("html-webpack-plugin");
+const { CleanWebpackPlugin } = require("clean-webpack-plugin");
+const TerserJSPlugin = require("terser-webpack-plugin");
+const CleanCss = require("clean-css");
+const OptimizeCSSAssetsPlugin = require("optimize-css-assets-webpack-plugin");
+const FaviconsWebpackPlugin = require("favicons-webpack-plugin");
+const MiniCssExtractPlugin = require("mini-css-extract-plugin");
+
+const { merge } = require("webpack-merge");
+
+const babelOptions = require("../babel/babel.prod");
+
+const CspHashPlugin = require("./cspHashPlugin");
+const sharedConfig = require("./webpack.config.shared");
+
+const fonts = path.resolve("src/fonts");
+const nodeModules = path.resolve("node_modules");
+
+const prodConfig = {
+  mode: "production",
+  bail: true,
+  cache: false,
+  output: {
+    filename: "static/[name]-[contenthash].js",
+  },
+  optimization: {
+    minimize: true,
+    minimizer: [
+      new TerserJSPlugin({}),
+      new OptimizeCSSAssetsPlugin({
+        cssProcessor: CleanCss,
+      }),
+    ],
+  },
+  devtool: "source-map",
+  module: {
+    rules: [
+      {
+        test: /\.jsx?$/,
+        loader: "babel-loader",
+        options: babelOptions,
+      },
+      {
+        test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i,
+        loader: "file-loader",
+        include: [nodeModules, fonts],
+        query: { name: "static/assets/[name]-[contenthash].[ext]" },
+      },
+    ],
+  },
+  plugins: [
+    new HtmlWebpackPlugin({
+      filename: "index.html",
+      template: path.resolve("index_template.html"),
+      decodeEntities: false,
+      minify: false,
+    }),
+    new CleanWebpackPlugin({
+      verbose: true,
+      protectWebpackAssets: false,
+      cleanAfterEveryBuildPatterns: ["main.js", "main.css"],
+    }),
+    new FaviconsWebpackPlugin({
+      logo: "./favicon.png",
+      prefix: "static/assets/",
+      favicons: {
+        icons: {
+          android: false,
+          appleIcon: false,
+          appleStartup: false,
+          coast: false,
+          firefox: false,
+          windows: false,
+          yandex: false,
+        },
+      },
+    }),
+    new MiniCssExtractPlugin({
+      filename: "static/[name]-[contenthash].css",
+    }),
+    new CspHashPlugin({
+      filename: "csp-hashes.json",
+    }),
+  ],
+  performance: {
+    maxEntrypointSize: 2000000,
+    maxAssetSize: 2000000,
+  },
+};
+
+module.exports = merge(sharedConfig, prodConfig);

client/configuration/webpack/webpack.config.shared.js

@@ -0,0 +1,74 @@
+const path = require("path");
+const fs = require("fs");
+const MiniCssExtractPlugin = require("mini-css-extract-plugin");
+const ObsoleteWebpackPlugin = require("obsolete-webpack-plugin");
+const ScriptExtHtmlWebpackPlugin = require("script-ext-html-webpack-plugin");
+
+const src = path.resolve("src");
+const nodeModules = path.resolve("node_modules");
+
+const publicPath = "/";
+
+const rawObsoleteHTMLTemplate = fs.readFileSync(
+  `${__dirname}/obsoleteHTMLTemplate.html`,
+  "utf8"
+);
+
+const obsoleteHTMLTemplate = rawObsoleteHTMLTemplate.replace(/'/g, '"');
+
+module.exports = {
+  entry: [
+    "core-js",
+    "regenerator-runtime/runtime",
+    "fastestsmallesttextencoderdecoder",
+    "whatwg-fetch",
+    "abort-controller/polyfill",
+    "./src/index",
+  ],
+  output: {
+    path: path.resolve("build"),
+    publicPath,
+  },
+  module: {
+    rules: [
+      {
+        test: /\.css$/,
+        include: src,
+        exclude: [path.resolve(src, "index.css")],
+        use: [
+          MiniCssExtractPlugin.loader,
+          {
+            loader: "css-loader",
+            options: {
+              modules: {
+                localIdentName: "[name]__[local]___[hash:base64:5]",
+              },
+              importLoaders: 1,
+            },
+          },
+        ],
+      },
+      {
+        test: /index\.css$/,
+        include: [path.resolve(src, "index.css")],
+        use: [MiniCssExtractPlugin.loader, "css-loader"],
+      },
+      {
+        test: /\.json$/,
+        include: [src, nodeModules],
+        loader: "json-loader",
+        exclude: /manifest.json$/,
+      },
+    ],
+  },
+  plugins: [
+    new ObsoleteWebpackPlugin({
+      name: "obsolete",
+      template: obsoleteHTMLTemplate,
+      promptOnNonTargetBrowser: false,
+    }),
+    new ScriptExtHtmlWebpackPlugin({
+      async: "obsolete",
+    }),
+  ],
+};

client/index.html

@@ -0,0 +1,46 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>cell&times;gene</title>
+    <style>
+      html,
+      body,
+      p,
+      h1,
+      h2,
+      h3,
+      h4,
+      h5,
+      h6,
+      span,
+      button,
+      input,
+      label,
+      text,
+      div {
+        font-family: "Roboto Condensed", "Helvetica Neue", "Helvetica", "Arial",
+          sans-serif;
+        font-size: 14px;
+      }
+      body {
+        margin: 0;
+        padding: 0;
+      }
+
+      * {
+        box-sizing: border-box;
+      }
+    </style>
+  </head>
+  <body>
+    <noscript
+      >If you're seeing this message, that means
+      <strong>JavaScript has been disabled on your browser</strong>, please
+      <strong>enable JS</strong> to make this app work.</noscript
+    >
+
+    <div id="root"></div>
+  </body>
+</html>

client/index_template.html

@@ -0,0 +1,65 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>cell&times;gene</title>
+    <style>
+      html,
+      body,
+      p,
+      h1,
+      h2,
+      h3,
+      h4,
+      h5,
+      h6,
+      span,
+      button,
+      input,
+      label,
+      text,
+      div {
+        font-family: "Roboto Condensed", "Helvetica Neue", "Helvetica", "Arial",
+          sans-serif;
+        font-size: 14px;
+      }
+
+      body {
+        margin: 0;
+        padding: 0;
+      }
+
+      * {
+        box-sizing: border-box;
+      }
+    </style>
+  </head>
+  <body>
+    <script type="text/javascript">
+      window.CELLXGENE = {};
+      window.CELLXGENE.API = {
+        prefix: `${location.origin}${location.pathname}api/`,
+        version: "v0.2/",
+      };
+    </script>
+    <noscript
+      >If you're seeing this message, that means
+      <strong>JavaScript has been disabled on your browser</strong>, please
+      <strong>enable JS</strong> to make this app work.
+    </noscript>
+
+    <div id="root"></div>
+    {% for script in SCRIPTS -%}
+    <script type="text/javascript"
+      {{ ('integrity="%s"' % script.integrity) | safe if script.integrity }}
+      {{ ('crossorigin="%s"' % script.crossorigin) | safe if script.crossorigin }}
+      src="{{script.src | safe}}"
+    ></script>
+    {%- endfor %}
+
+    {% for ils in INLINE_SCRIPTS %}
+    <script type="text/javascript" no-csp-hash>{% include ils %}</script>
+    {% endfor %}
+  </body>
+</html>

client/jest-puppeteer.config.js

@@ -0,0 +1,46 @@
+/**
+ * `client/jest-puppeteer.config.js` is for configuring Puppeteer's launch config options
+ * `client/__tests__/e2e/puppeteer.setup.js` is for configuring `jest`, `browser`,
+ * and `page` objects
+ */
+
+const ENV_DEFAULT = require("../environment.default.json");
+
+const jestEnv = process.env.JEST_ENV || ENV_DEFAULT.JEST_ENV;
+const isHeadful =
+  process.env.HEADFUL === "true" || process.env.HEADLESS === "false";
+
+const DEFAULT_LAUNCH_CONFIG = {
+  headless: !isHeadful,
+  args: ["--ignore-certificate-errors", "--ignore-ssl-errors"],
+  ignoreHTTPSErrors: true,
+  defaultViewport: {
+    width: 1280,
+    height: 960,
+  },
+};
+
+const LAUNCH_CONFIG_BY_ENV = {
+  [ENV_DEFAULT.DEBUG]: {
+    ...DEFAULT_LAUNCH_CONFIG,
+    headless: false,
+    slowMo: 100,
+    devtools: true,
+    defaultViewport: {
+      width: DEFAULT_LAUNCH_CONFIG.defaultViewport.width,
+      height: DEFAULT_LAUNCH_CONFIG.defaultViewport.height + 560,
+    },
+  },
+  [ENV_DEFAULT.DEV]: {
+    ...DEFAULT_LAUNCH_CONFIG,
+    headless: false,
+    slowMo: 5,
+  },
+};
+
+const launchConfig = LAUNCH_CONFIG_BY_ENV[jestEnv] || DEFAULT_LAUNCH_CONFIG;
+
+module.exports = {
+  browserContext: "incognito",
+  launch: launchConfig,
+};

client/package.json

@@ -0,0 +1,185 @@
+{
+  "name": "cellxgene",
+  "version": "0.16.0",
+  "license": "MIT",
+  "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
+  "repository": "https://github.com/chanzuckerberg/cellxgene",
+  "scripts": {
+    "build": "npm run clean && webpack --config",
+    "clean": "rimraf build",
+    "dev": "npm run build -- configuration/webpack/webpack.config.dev.js",
+    "e2e": "jest --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
+    "e2e-annotations": "jest --config __tests__/e2e/e2eJestConfig.json e2e/e2eAnnotations.test.js",
+    "e2e-prod": "CXG_URL_BASE='https://cellxgene.cziscience.com/d/pbmc3k.cxg/' jest --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
+    "fmt": "eslint --fix src __tests__",
+    "lint": "eslint --fix src __tests__",
+    "prod": "npm run build -- configuration/webpack/webpack.config.prod.js",
+    "test": "jest --testPathIgnorePatterns e2e"
+  },
+  "engineStrict": true,
+  "engines": {
+    "npm": ">=3.0.0"
+  },
+  "eslintConfig": {
+    "extends": "./configuration/eslint/eslint.js"
+  },
+  "eslintIgnore": [
+    "src/util/stateManager/matrix_generated.js"
+  ],
+  "resolutions": {
+    "eslint-scope": "3.7.1"
+  },
+  "browserslist": [
+    "Chrome > 60",
+    "Safari >= 10.1",
+    "iOS >= 10.3",
+    "Firefox >= 60",
+    "Edge >= 15",
+    "not Explorer > 0"
+  ],
+  "dependencies": {
+    "@blueprintjs/core": "^3.30.0",
+    "@blueprintjs/icons": "^3.19.0",
+    "@blueprintjs/select": "^3.13.5",
+    "abort-controller": "^3.0.0",
+    "core-js": "^3.6.5",
+    "d3": "^4.10.0",
+    "d3-scale-chromatic": "^1.5.0",
+    "fastestsmallesttextencoderdecoder": "^1.0.22",
+    "flatbuffers": "^1.11.0",
+    "fuzzysort": "^1.1.4",
+    "gl-mat4": "^1.2.0",
+    "gl-matrix": "^3.3.0",
+    "gl-vec3": "^1.1.3",
+    "is-number": "^7.0.0",
+    "lodash": "^4.17.19",
+    "memoize-one": "^5.1.1",
+    "react": "^16.13.1",
+    "react-async": "^10.0.1",
+    "react-dom": "^16.13.1",
+    "react-flip-toolkit": "^7.0.12",
+    "react-helmet": "^5.2.1",
+    "react-icons": "^3.10.0",
+    "react-redux": "^7.2.0",
+    "redux": "^4.0.5",
+    "redux-thunk": "^2.3.0",
+    "regenerator-runtime": "^0.13.7",
+    "regl": "^1.6.1",
+    "script-ext-html-webpack-plugin": "^2.1.4",
+    "tinyqueue": "^2.0.3",
+    "webpack-merge": "^5.0.9",
+    "whatwg-fetch": "^3.2.0"
+  },
+  "devDependencies": {
+    "@babel/core": "^7.10.5",
+    "@babel/plugin-proposal-class-properties": "^7.10.4",
+    "@babel/plugin-proposal-decorators": "^7.10.5",
+    "@babel/plugin-proposal-export-namespace-from": "^7.10.4",
+    "@babel/plugin-proposal-function-bind": "^7.10.5",
+    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.10.4",
+    "@babel/plugin-proposal-optional-chaining": "^7.10.4",
+    "@babel/plugin-transform-react-constant-elements": "^7.10.4",
+    "@babel/plugin-transform-runtime": "^7.10.5",
+    "@babel/preset-env": "^7.10.4",
+    "@babel/preset-react": "^7.10.4",
+    "@babel/register": "^7.10.5",
+    "@babel/runtime": "^7.10.5",
+    "@sentry/webpack-plugin": "^1.12.0",
+    "babel-eslint": "^10.1.0",
+    "babel-jest": "^26.1.0",
+    "babel-loader": "^8.1.0",
+    "babel-preset-modern-browsers": "^14.2.1",
+    "chalk": "^4.1.0",
+    "cheerio": "^1.0.0-rc.3",
+    "clean-css": "^4.2.3",
+    "clean-webpack-plugin": "^3.0.0",
+    "codecov": "^3.7.1",
+    "connect-history-api-fallback": "^1.6.0",
+    "css-loader": "^3.6.0",
+    "eslint": "^7.4.0",
+    "eslint-config-airbnb": "^18.2.0",
+    "eslint-config-prettier": "^6.11.0",
+    "eslint-loader": "^3.0.4",
+    "eslint-plugin-compat": "^3.8.0",
+    "eslint-plugin-eslint-comments": "^3.2.0",
+    "eslint-plugin-filenames": "^1.3.2",
+    "eslint-plugin-import": "^2.22.0",
+    "eslint-plugin-jest": "^23.18.0",
+    "eslint-plugin-jsx-a11y": "^6.3.1",
+    "eslint-plugin-prettier": "^3.1.4",
+    "eslint-plugin-react": "^7.20.3",
+    "eslint-plugin-react-hooks": "^4.0.8",
+    "expect-puppeteer": "^4.4.0",
+    "express": "^4.17.1",
+    "favicons-webpack-plugin": "^3.0.1",
+    "file-loader": "^6.0.0",
+    "html-webpack-plugin": "^4.3.0",
+    "husky": "^4.2.5",
+    "jest": "^26.1.0",
+    "jest-circus": "^26.1.0",
+    "jest-environment-puppeteer": "^4.4.0",
+    "jest-fetch-mock": "^3.0.3",
+    "jest-puppeteer": "^4.4.0",
+    "json-loader": "^0.5.7",
+    "lint-staged": "^10.2.11",
+    "mini-css-extract-plugin": "^0.9.0",
+    "obsolete-webpack-plugin": "^0.5.6",
+    "optimize-css-assets-webpack-plugin": "^5.0.3",
+    "prettier": "^2.0.5",
+    "puppeteer": "^3.3.0",
+    "rimraf": "^3.0.2",
+    "serve-favicon": "^2.5.0",
+    "style-loader": "^1.2.1",
+    "sw-precache-webpack-plugin": "^1.0.0",
+    "terser-webpack-plugin": "^3.0.7",
+    "url-loader": "^4.1.0",
+    "webpack": "^4.43.0",
+    "webpack-cli": "^3.3.12",
+    "webpack-dev-middleware": "^3.7.2"
+  },
+  "jest": {
+    "testMatch": [
+      "**/__tests__/**/?(*.)(spec|test).js?(x)"
+    ],
+    "setupFiles": [
+      "./__tests__/setupMissingGlobals.js"
+    ],
+    "coverageDirectory": "./coverage/",
+    "collectCoverage": true
+  },
+  "babel": {
+    "env": {
+      "test": {
+        "presets": [
+          "@babel/preset-env",
+          "@babel/preset-react"
+        ],
+        "plugins": [
+          "@babel/plugin-proposal-function-bind",
+          [
+            "@babel/plugin-proposal-decorators",
+            {
+              "legacy": true
+            }
+          ],
+          [
+            "@babel/plugin-proposal-class-properties",
+            {
+              "loose": true
+            }
+          ],
+          "@babel/plugin-proposal-export-namespace-from",
+          "@babel/plugin-transform-react-constant-elements",
+          "@babel/plugin-transform-runtime",
+          "@babel/plugin-proposal-optional-chaining",
+          "@babel/plugin-proposal-nullish-coalescing-operator"
+        ]
+      }
+    }
+  },
+  "husky": {
+    "hooks": {
+      "pre-commit": "lint-staged --config \"./configuration/lint-staged/lint-staged.config.js\""
+    }
+  }
+}

client/server/development.js

@@ -11,7 +11,7 @@ var utils = require("./utils");
 
 process.env.NODE_ENV = "development";
 
-var PORT = process.env.PORT || 3000;
+const CLIENT_PORT = process.env.CXG_CLIENT_PORT;
 
 // Set up compiler
 var compiler = webpack(config);
@@ -21,8 +21,8 @@ compiler.plugin("invalid", () => {
   console.log("Compiling...");
 });
 
-compiler.plugin("done", stats => {
+compiler.plugin("done", (stats) => {
-  utils.formatStats(stats, PORT);
+  utils.formatStats(stats, CLIENT_PORT);
 });
 
 // Launch server
@@ -32,20 +32,18 @@ app.use(historyApiFallback({ verbose: false }));
 
 app.use(
   require("webpack-dev-middleware")(compiler, {
-    noInfo: true,
+    logLevel: "warn",
-    publicPath: config.output.publicPath
+    publicPath: config.output.publicPath,
   })
 );
 
-app.use(require("webpack-hot-middleware")(compiler));
-
 app.use(favicon("./favicon.png"));
 
 app.get("*", (req, res) => {
   res.sendFile(path.resolve("index.html"));
 });
 
-app.listen(PORT, err => {
+app.listen(CLIENT_PORT, (err) => {
   if (err) {
     console.log(err);
     return;

client/server/utils.js

@@ -36,10 +36,10 @@ var formatStats = (stats, port) => {
 
   var json = stats.toJson();
   var formattedErrors = json.errors.map(
-    message => "Error in " + formatMessage(message)
+    (message) => "Error in " + formatMessage(message)
   );
   var formattedWarnings = json.warnings.map(
-    message => "Warning in " + formatMessage(message)
+    (message) => "Warning in " + formatMessage(message)
   );
 
   if (hasErrors) {
@@ -48,7 +48,7 @@ var formatStats = (stats, port) => {
     if (formattedErrors.some(isLikelyASyntaxError)) {
       formattedErrors = formattedErrors.filter(isLikelyASyntaxError);
     }
-    formattedErrors.forEach(message => {
+    formattedErrors.forEach((message) => {
       console.log(message);
       console.log();
     });
@@ -58,7 +58,7 @@ var formatStats = (stats, port) => {
   if (hasWarnings) {
     console.log(chalk.yellow("Compiled with warnings."));
     console.log();
-    formattedWarnings.forEach(message => {
+    formattedWarnings.forEach((message) => {
       console.log(message);
       console.log();
     });

client/src/actions/annotation.js

@@ -0,0 +1,387 @@
+/*
+Action creators for user annotation
+*/
+import _ from "lodash";
+import * as globals from "../globals";
+import { MatrixFBS, AnnotationsHelpers } from "../util/stateManager";
+
+const { isUserAnnotation } = AnnotationsHelpers;
+
+export const annotationCreateCategoryAction = (
+  newCategoryName,
+  categoryToDuplicate
+) => async (dispatch, getState) => {
+  /*
+  Add a new user-created category to the obs annotations.
+
+  Arguments:
+    newCategoryName - string name for the category.
+    categoryToDuplicate - obs category to use for initial values, or null.
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
+  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+  const { schema } = prevAnnoMatrix;
+
+  /* name must be a string,  non-zero length */
+  if (typeof newCategoryName !== "string" || newCategoryName.length === 0)
+    throw new Error("user annotations require string name");
+
+  /* ensure the name isn't already in use! */
+  if (schema.annotations.obsByName[newCategoryName])
+    throw new Error("name collision on annotation category create");
+
+  let initialValue;
+  let newSchema;
+  let ctor;
+  if (categoryToDuplicate) {
+    /* if we are duplicating a category, retrieve it */
+    const catDupSchema = schema.annotations.obsByName[categoryToDuplicate];
+    const catDupType = catDupSchema?.type;
+    if (catDupType !== "string" && catDupType !== "categorical")
+      throw new Error("categoryToDuplicate does not exist or has invalid type");
+
+    const catToDupDf = await prevAnnoMatrix
+      .base()
+      .fetch("obs", categoryToDuplicate);
+    const col = catToDupDf.col(categoryToDuplicate);
+    initialValue = col.asArray();
+    const { categories } = col.summarizeCategorical();
+    // all user-created annotations must have the unassigned category
+    if (!categories.includes(globals.unassignedCategoryLabel)) {
+      categories.push(globals.unassignedCategoryLabel);
+    }
+    ctor = initialValue.constructor;
+    newSchema = {
+      ...catDupSchema,
+      name: newCategoryName,
+      categories,
+      writable: true,
+    };
+  } else {
+    /* else assign to the standard default value */
+    initialValue = globals.unassignedCategoryLabel;
+    ctor = Array;
+    newSchema = {
+      name: newCategoryName,
+      type: "categorical",
+      categories: [globals.unassignedCategoryLabel],
+      writable: true,
+    };
+  }
+
+  const obsCrossfilter = prevObsCrossfilter.addObsColumn(
+    newSchema,
+    ctor,
+    initialValue
+  );
+
+  dispatch({
+    type: "annotation: create category",
+    data: newCategoryName,
+    categoryToDuplicate,
+    annoMatrix: obsCrossfilter.annoMatrix,
+    obsCrossfilter,
+  });
+};
+
+export const annotationRenameCategoryAction = (
+  oldCategoryName,
+  newCategoryName
+) => (dispatch, getState) => {
+  /*
+  Rename a user-created annotation category
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
+  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+  if (!isUserAnnotation(prevAnnoMatrix, oldCategoryName))
+    throw new Error("not a user annotation");
+
+  /* name must be a string,  non-zero length */
+  if (typeof newCategoryName !== "string" || newCategoryName.length === 0)
+    throw new Error("user annotations require string name");
+
+  if (oldCategoryName === newCategoryName) return;
+
+  const obsCrossfilter = prevObsCrossfilter.renameObsColumn(
+    oldCategoryName,
+    newCategoryName
+  );
+
+  dispatch({
+    type: "annotation: category edited",
+    annoMatrix: obsCrossfilter.annoMatrix,
+    obsCrossfilter,
+    metadataField: oldCategoryName,
+    newCategoryText: newCategoryName,
+    data: newCategoryName,
+  });
+};
+
+export const annotationDeleteCategoryAction = (categoryName) => (
+  dispatch,
+  getState
+) => {
+  /*
+  Delete a user-created category
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
+  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+    throw new Error("not a user annotation");
+
+  const obsCrossfilter = prevObsCrossfilter.dropObsColumn(categoryName);
+  dispatch({
+    type: "annotation: delete category",
+    annoMatrix: obsCrossfilter.annoMatrix,
+    obsCrossfilter,
+    metadataField: categoryName,
+  });
+};
+
+export const annotationCreateLabelInCategory = (
+  categoryName,
+  labelName,
+  assignSelected
+) => async (dispatch, getState) => {
+  /*
+  Add a new label to a user-defined category.  If assignSelected is true, assign 
+  the label to all currently selected cells.
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
+  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+    throw new Error("not a user annotation");
+
+  let obsCrossfilter = prevObsCrossfilter.addObsAnnoCategory(
+    categoryName,
+    labelName
+  );
+  if (assignSelected) {
+    obsCrossfilter = await obsCrossfilter.setObsColumnValues(
+      categoryName,
+      prevObsCrossfilter.allSelectedLabels(),
+      labelName
+    );
+  }
+
+  dispatch({
+    type: "annotation: add new label to category",
+    annoMatrix: obsCrossfilter.annoMatrix,
+    obsCrossfilter,
+    metadataField: categoryName,
+    newLabelText: labelName,
+    assignSelectedCells: assignSelected,
+  });
+};
+
+export const annotationDeleteLabelFromCategory = (
+  categoryName,
+  labelName
+) => async (dispatch, getState) => {
+  /*
+  delete a label from a user-defined category
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
+  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+    throw new Error("not a user annotation");
+
+  const obsCrossfilter = await prevObsCrossfilter.removeObsAnnoCategory(
+    categoryName,
+    labelName,
+    globals.unassignedCategoryLabel
+  );
+
+  dispatch({
+    type: "annotation: delete label",
+    metadataField: categoryName,
+    label: labelName,
+    annoMatrix: obsCrossfilter.annoMatrix,
+    obsCrossfilter,
+  });
+};
+
+export const annotationRenameLabelInCategory = (
+  categoryName,
+  oldLabelName,
+  newLabelName
+) => async (dispatch, getState) => {
+  /*
+  label name change
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
+  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+    throw new Error("not a user annotation");
+
+  let obsCrossfilter = await prevObsCrossfilter.resetObsColumnValues(
+    categoryName,
+    oldLabelName,
+    newLabelName
+  );
+  obsCrossfilter = await obsCrossfilter.removeObsAnnoCategory(
+    categoryName,
+    oldLabelName,
+    globals.unassignedCategoryLabel
+  );
+
+  dispatch({
+    type: "annotation: label edited",
+    editedLabel: newLabelName,
+    metadataField: categoryName,
+    label: oldLabelName,
+    annoMatrix: obsCrossfilter.annoMatrix,
+    obsCrossfilter,
+  });
+};
+
+export const annotationLabelCurrentSelection = (
+  categoryName,
+  labelName
+) => async (dispatch, getState) => {
+  /*
+  set the label on all currently selected
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
+  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+    throw new Error("not a user annotation");
+
+  const obsCrossfilter = await prevObsCrossfilter.setObsColumnValues(
+    categoryName,
+    prevObsCrossfilter.allSelectedLabels(),
+    labelName
+  );
+
+  dispatch({
+    type: "annotation: label current cell selection",
+    metadataField: categoryName,
+    label: labelName,
+    obsCrossfilter,
+    annoMatrix: obsCrossfilter.annoMatrix,
+  });
+};
+
+function writableAnnotations(annoMatrix) {
+  return annoMatrix.schema.annotations.obs.columns
+    .filter((s) => s.writable)
+    .map((s) => s.name);
+}
+
+export const needToSaveObsAnnotations = (annoMatrix, lastSavedAnnoMatrix) => {
+  /*
+  Return true if there are LIKELY user-defined annotation modifications between the two
+  annoMatrices.  Technically not an action creator, but intimately intertwined
+  with the save process.
+
+  Two conditions will trigger a need to save:
+    * the collection of user-defined columns have changed
+    * the contents of the user-defined columns have change
+  */
+
+  annoMatrix = annoMatrix.base();
+
+  // if the annoMatrix hasn't changed, we are guaranteed no changes to the matrix schema or contents.
+  if (annoMatrix === lastSavedAnnoMatrix) return false;
+
+  // if the schema has changed, we need to save
+  const currentWritable = writableAnnotations(annoMatrix);
+  if (_.difference(currentWritable, writableAnnotations(lastSavedAnnoMatrix))) {
+    return true;
+  }
+
+  // no schema changes; check for change in contents
+  return currentWritable.some(
+    (col) => annoMatrix.col(col) !== lastSavedAnnoMatrix.col(col)
+  );
+};
+
+export const saveObsAnnotationsAction = () => async (dispatch, getState) => {
+  /*
+  Save the user-created obs annotations IF any have changed.
+  */
+  const state = getState();
+  const { annotations, autosave } = state;
+  const { dataCollectionNameIsReadOnly, dataCollectionName } = annotations;
+  const { lastSavedAnnoMatrix, saveInProgress } = autosave;
+
+  const annoMatrix = state.annoMatrix.base();
+
+  if (saveInProgress || annoMatrix === lastSavedAnnoMatrix) return;
+  if (!needToSaveObsAnnotations(annoMatrix, lastSavedAnnoMatrix)) {
+    dispatch({
+      type: "writable obs annotations - save complete",
+      lastSavedAnnoMatrix: annoMatrix,
+    });
+    return;
+  }
+
+  /*
+  Else, we really do need to save
+  */
+
+  dispatch({
+    type: "writable obs annotations - save started",
+  });
+
+  const df = await annoMatrix.fetch("obs", writableAnnotations(annoMatrix));
+  const matrix = MatrixFBS.encodeMatrixFBS(df);
+  try {
+    const queryString =
+      !dataCollectionNameIsReadOnly && !!dataCollectionName
+        ? `?annotation-collection-name=${encodeURIComponent(
+            dataCollectionName
+          )}`
+        : "";
+    const res = await fetch(
+      `${globals.API.prefix}${globals.API.version}annotations/obs${queryString}`,
+      {
+        method: "PUT",
+        body: matrix,
+        headers: new Headers({
+          "Content-Type": "application/octet-stream",
+        }),
+        credentials: "include",
+      }
+    );
+    if (res.ok) {
+      dispatch({
+        type: "writable obs annotations - save complete",
+        lastSavedAnnoMatrix: annoMatrix,
+      });
+    } else {
+      dispatch({
+        type: "writable obs annotations - save error",
+        message: `HTTP error ${res.status} - ${res.statusText}`,
+        res,
+      });
+    }
+  } catch (error) {
+    dispatch({
+      type: "writable obs annotations - save error",
+      message: error.toString(),
+      error,
+    });
+  }
+};

client/src/actions/embedding.js

@@ -0,0 +1,44 @@
+/*
+action creators related to embeddings choice
+*/
+
+import { AnnoMatrixObsCrossfilter } from "../annoMatrix";
+import { _setEmbeddingSubset } from "../util/stateManager/viewStackHelpers";
+
+export async function _switchEmbedding(prevAnnoMatrix, newEmbeddingName) {
+  /*
+  DRY helper used by this and reembedding action creators
+  */
+  const base = prevAnnoMatrix.base();
+  const embeddingDf = await base.fetch("emb", newEmbeddingName);
+  const annoMatrix = _setEmbeddingSubset(prevAnnoMatrix, embeddingDf);
+  const obsCrossfilter = await new AnnoMatrixObsCrossfilter(annoMatrix).select(
+    "emb",
+    newEmbeddingName,
+    {
+      mode: "all",
+    }
+  );
+  return [annoMatrix, obsCrossfilter];
+}
+
+export const layoutChoiceAction = (newLayoutChoice) => async (
+  dispatch,
+  getState
+) => {
+  /*
+  On layout choice, make sure we have selected all on the previous layout, AND the new
+  layout.
+  */
+  const { annoMatrix: prevAnnoMatrix } = getState();
+  const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
+    prevAnnoMatrix,
+    newLayoutChoice
+  );
+  dispatch({
+    type: "set layout choice",
+    layoutChoice: newLayoutChoice,
+    obsCrossfilter,
+    annoMatrix,
+  });
+};

client/src/actions/index.js

@@ -0,0 +1,236 @@
+import * as globals from "../globals";
+import { AnnoMatrixLoader, AnnoMatrixObsCrossfilter } from "../annoMatrix";
+import {
+  catchErrorsWrap,
+  doJsonRequest,
+  dispatchNetworkErrorMessageToUser,
+} from "../util/actionHelpers";
+import {
+  requestReembed /* , reembedResetWorldToUniverse -- disabled temporarily, TODO issue #1606 */,
+} from "./reembed";
+import { loadUserColorConfig } from "../util/stateManager/colorHelpers";
+import * as selnActions from "./selection";
+import * as annoActions from "./annotation";
+import * as viewActions from "./viewStack";
+import * as embActions from "./embedding";
+
+/*
+return promise fetching user-configured colors
+*/
+async function userColorsFetchAndLoad(dispatch) {
+  return fetchJson("colors").then((response) =>
+    dispatch({
+      type: "universe: user color load success",
+      userColors: loadUserColorConfig(response),
+    })
+  );
+}
+
+async function schemaFetch() {
+  return fetchJson("schema");
+}
+
+async function configFetch(dispatch) {
+  return fetchJson("config").then((response) => {
+    const config = { ...globals.configDefaults, ...response.config };
+    dispatch({
+      type: "configuration load complete",
+      config,
+    });
+    return config;
+  });
+}
+
+function prefetchEmbeddings(annoMatrix) {
+  /*
+  prefetch requests for all embeddings
+  */
+  const { schema } = annoMatrix;
+  const available = schema.layout.obs.map((v) => v.name);
+  available.forEach((embName) => annoMatrix.prefetch("emb", embName));
+}
+
+/*
+Application bootstrap
+*/
+const doInitialDataLoad = () =>
+  catchErrorsWrap(async (dispatch) => {
+    dispatch({ type: "initial data load start" });
+
+    try {
+      const [config, schema] = await Promise.all([
+        configFetch(dispatch),
+        schemaFetch(dispatch),
+        userColorsFetchAndLoad(dispatch),
+      ]);
+
+      const baseDataUrl = `${globals.API.prefix}${globals.API.version}`;
+      const annoMatrix = new AnnoMatrixLoader(baseDataUrl, schema.schema);
+      const obsCrossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
+      prefetchEmbeddings(annoMatrix);
+
+      dispatch({
+        type: "annoMatrix: init complete",
+        annoMatrix,
+        obsCrossfilter,
+      });
+      dispatch({ type: "initial data load complete" });
+
+      const defaultEmbedding = config?.parameters?.["default_embedding"];
+      const layoutSchema = schema?.schema?.layout?.obs ?? [];
+      if (
+        defaultEmbedding &&
+        layoutSchema.some((s) => s.name === defaultEmbedding)
+      ) {
+        dispatch(embActions.layoutChoiceAction(defaultEmbedding));
+      }
+    } catch (error) {
+      dispatch({ type: "initial data load error", error });
+    }
+  }, true);
+
+function requestSingleGeneExpressionCountsForColoringPOST(gene) {
+  return {
+    type: "color by expression",
+    gene,
+  };
+}
+
+const requestUserDefinedGene = (gene) => ({
+  type: "request user defined gene success",
+  data: {
+    genes: [gene],
+  },
+});
+
+const dispatchDiffExpErrors = (dispatch, response) => {
+  switch (response.status) {
+    case 403:
+      dispatchNetworkErrorMessageToUser(
+        "Too many cells selected for differential experesion calculation - please make a smaller selection."
+      );
+      break;
+    case 501:
+      dispatchNetworkErrorMessageToUser(
+        "Differential expression is not implemented."
+      );
+      break;
+    default: {
+      const msg = `Unexpected differential expression HTTP response ${response.status}, ${response.statusText}`;
+      dispatchNetworkErrorMessageToUser(msg);
+      dispatch({
+        type: "request differential expression error",
+        error: new Error(msg),
+      });
+    }
+  }
+};
+
+const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
+  dispatch,
+  getState
+) => {
+  dispatch({ type: "request differential expression started" });
+  try {
+    /*
+    Steps:
+    1. get the most differentially expressed genes
+    2. get expression data for each
+    */
+    const { annoMatrix } = getState();
+    const varIndexName = annoMatrix.schema.annotations.var.index;
+
+    // Legal values are null, Array or TypedArray.  Null is initial state.
+    if (!set1) set1 = [];
+    if (!set2) set2 = [];
+
+    // These lines ensure that we convert any TypedArray to an Array.
+    // This is necessary because JSON.stringify() does some very strange
+    // things with TypedArrays (they are marshalled to JSON objects, rather
+    // than being marshalled as a JSON array).
+    set1 = Array.isArray(set1) ? set1 : Array.from(set1);
+    set2 = Array.isArray(set2) ? set2 : Array.from(set2);
+
+    const res = await fetch(
+      `${globals.API.prefix}${globals.API.version}diffexp/obs`,
+      {
+        method: "POST",
+        headers: new Headers({
+          Accept: "application/json",
+          "Content-Type": "application/json",
+        }),
+        body: JSON.stringify({
+          mode: "topN",
+          count: num_genes,
+          set1: { filter: { obs: { index: set1 } } },
+          set2: { filter: { obs: { index: set2 } } },
+        }),
+        credentials: "include",
+      }
+    );
+
+    if (!res.ok || res.headers.get("Content-Type") !== "application/json") {
+      return dispatchDiffExpErrors(dispatch, res);
+    }
+
+    const response = await res.json();
+    const varIndex = await annoMatrix.fetch("var", varIndexName);
+    const data = response.map((v) => [
+      varIndex.at(v[0], varIndexName),
+      ...v.slice(1),
+    ]);
+
+    /* then send the success case action through */
+    return dispatch({
+      type: "request differential expression success",
+      data,
+    });
+  } catch (error) {
+    return dispatch({
+      type: "request differential expression error",
+      error,
+    });
+  }
+};
+
+function fetchJson(pathAndQuery) {
+  return doJsonRequest(
+    `${globals.API.prefix}${globals.API.version}${pathAndQuery}`
+  );
+}
+
+export default {
+  doInitialDataLoad,
+  requestDifferentialExpression,
+  requestSingleGeneExpressionCountsForColoringPOST,
+  requestUserDefinedGene,
+  requestReembed,
+  selectContinuousMetadataAction: selnActions.selectContinuousMetadataAction,
+  selectCategoricalMetadataAction: selnActions.selectCategoricalMetadataAction,
+  selectCategoricalAllMetadataAction:
+    selnActions.selectCategoricalAllMetadataAction,
+  graphBrushStartAction: selnActions.graphBrushStartAction,
+  graphBrushChangeAction: selnActions.graphBrushChangeAction,
+  graphBrushDeselectAction: selnActions.graphBrushDeselectAction,
+  graphBrushCancelAction: selnActions.graphBrushCancelAction,
+  graphBrushEndAction: selnActions.graphBrushEndAction,
+  graphLassoStartAction: selnActions.graphLassoStartAction,
+  graphLassoEndAction: selnActions.graphLassoEndAction,
+  graphLassoCancelAction: selnActions.graphLassoCancelAction,
+  graphLassoDeselectAction: selnActions.graphLassoDeselectAction,
+  clipAction: viewActions.clipAction,
+  subsetAction: viewActions.subsetAction,
+  resetSubsetAction: viewActions.resetSubsetAction,
+  annotationCreateCategoryAction: annoActions.annotationCreateCategoryAction,
+  annotationRenameCategoryAction: annoActions.annotationRenameCategoryAction,
+  annotationDeleteCategoryAction: annoActions.annotationDeleteCategoryAction,
+  annotationCreateLabelInCategory: annoActions.annotationCreateLabelInCategory,
+  annotationDeleteLabelFromCategory:
+    annoActions.annotationDeleteLabelFromCategory,
+  annotationRenameLabelInCategory: annoActions.annotationRenameLabelInCategory,
+  annotationLabelCurrentSelection: annoActions.annotationLabelCurrentSelection,
+  saveObsAnnotationsAction: annoActions.saveObsAnnotationsAction,
+  needToSaveObsAnnotations: annoActions.needToSaveObsAnnotations,
+  layoutChoiceAction: embActions.layoutChoiceAction,
+  setCellSetFromSelection: selnActions.setCellSetFromSelection,
+};

client/src/actions/reembed.js

@@ -0,0 +1,108 @@
+import { API } from "../globals";
+import {
+  postNetworkErrorToast,
+  postAsyncSuccessToast,
+  postAsyncFailureToast,
+} from "../components/framework/toasters";
+import { _switchEmbedding } from "./embedding";
+
+function abortableFetch(request, opts, timeout = 0) {
+  const controller = new AbortController();
+  const { signal } = controller;
+
+  return {
+    abort: () => controller.abort(),
+    isAborted: () => signal.aborted,
+    ready: () => {
+      if (timeout) {
+        setTimeout(() => controller.abort(), timeout);
+      }
+      return fetch(request, { ...opts, signal });
+    },
+  };
+}
+
+async function doReembedFetch(dispatch, getState) {
+  const state = getState();
+  let cells = state.annoMatrix.rowIndex.labels();
+
+  // These lines ensure that we convert any TypedArray to an Array.
+  // This is necessary because JSON.stringify() does some very strange
+  // things with TypedArrays (they are marshalled to JSON objects, rather
+  // than being marshalled as a JSON array).
+  cells = Array.isArray(cells) ? cells : Array.from(cells);
+
+  const af = abortableFetch(
+    `${API.prefix}${API.version}layout/obs`,
+    {
+      method: "PUT",
+      headers: new Headers({
+        Accept: "application/octet-stream",
+        "Content-Type": "application/json",
+      }),
+      body: JSON.stringify({
+        method: "umap",
+        filter: { obs: { index: cells } },
+      }),
+      credentials: "include",
+    },
+    60000 // 1 minute timeout
+  );
+  dispatch({
+    type: "reembed: request start",
+    abortableFetch: af,
+  });
+  const res = await af.ready();
+
+  if (res.ok && res.headers.get("Content-Type").includes("application/json")) {
+    return res;
+  }
+
+  // else an error
+  let msg = `Unexpected HTTP response ${res.status}, ${res.statusText}`;
+  const body = await res.text();
+  if (body && body.length > 0) {
+    msg = `${msg} -- ${body}`;
+  }
+  throw new Error(msg);
+}
+
+/*
+functions below are dispatch-able
+*/
+export function requestReembed() {
+  return async (dispatch, getState) => {
+    try {
+      const res = await doReembedFetch(dispatch, getState);
+      const schema = await res.json();
+      dispatch({
+        type: "reembed: request completed",
+      });
+
+      const { annoMatrix: prevAnnoMatrix } = getState();
+      const base = prevAnnoMatrix.base().addEmbedding(schema);
+      const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
+        base,
+        schema.name
+      );
+      dispatch({
+        type: "reembed: add reembedding",
+        schema,
+        annoMatrix,
+        obsCrossfilter,
+      });
+
+      postAsyncSuccessToast("Re-embedding has completed.");
+    } catch (error) {
+      dispatch({
+        type: "reembed: request aborted",
+      });
+      if (error.name === "AbortError") {
+        postAsyncFailureToast("Re-embedding calculation was aborted.");
+      } else {
+        postNetworkErrorToast(`Re-embedding: ${error.message}`);
+      }
+      console.log("Reembed exception:", error, error.name, error.message);
+    }
+  };
+}

client/src/actions/selection.js

@@ -0,0 +1,199 @@
+/*
+Action creators for selection 
+*/
+export const selectContinuousMetadataAction = (
+  type,
+  query,
+  range,
+  oldProps = {}
+) => async (dispatch, getState) => {
+  const { obsCrossfilter: prevObsCrossfilter } = getState();
+
+  const selection = range
+    ? {
+        mode: "range",
+        lo: range[0],
+        hi: range[1],
+        inclusive: true, // [lo, hi] incluisve selection
+      }
+    : { mode: "all" };
+
+  const obsCrossfilter = await prevObsCrossfilter.select(...query, selection);
+
+  dispatch({
+    type,
+    obsCrossfilter,
+    range,
+    ...oldProps,
+  });
+};
+
+export const selectCategoricalMetadataAction = (
+  type, // action type
+  metadataField, // annotation category name
+  labels,
+  label, // the label being selected/deselected
+  isSelected, // bool
+  oldProps = {}
+) => async (dispatch, getState) => {
+  const {
+    obsCrossfilter: prevObsCrossfilter,
+    categoricalSelection,
+  } = getState();
+
+  const labelSelectionState = new Map(categoricalSelection[metadataField]);
+  labels.forEach(
+    (l) => labelSelectionState.has(l) || labelSelectionState.set(l, true)
+  );
+  labelSelectionState.set(label, isSelected);
+
+  const values = Array.from(labelSelectionState.keys()).filter((k) =>
+    labelSelectionState.get(k)
+  );
+  const selection = {
+    mode: "exact",
+    values,
+  };
+  const obsCrossfilter = await prevObsCrossfilter.select(
+    "obs",
+    metadataField,
+    selection
+  );
+
+  dispatch({
+    type,
+    obsCrossfilter,
+    metadataField,
+    labelSelectionState,
+    ...oldProps,
+  });
+};
+
+export const selectCategoricalAllMetadataAction = (
+  type, // action type
+  metadataField, // annotation category name
+  labels,
+  isSelected, // bool, select all or none
+  oldProps = {}
+) => async (dispatch, getState) => {
+  const {
+    obsCrossfilter: prevObsCrossfilter,
+    categoricalSelection,
+  } = getState();
+
+  const labelSelectionState = new Map(categoricalSelection[metadataField]);
+  labels.forEach((label) => labelSelectionState.set(label, isSelected));
+
+  const selection = { mode: isSelected ? "all" : "none" };
+  const obsCrossfilter = await prevObsCrossfilter.select(
+    "obs",
+    metadataField,
+    selection
+  );
+
+  dispatch({
+    type,
+    obsCrossfilter,
+    metadataField,
+    labelSelectionState,
+    ...oldProps,
+  });
+};
+
+/**
+ ** Graph selection-related actions
+ **/
+
+export const graphBrushStartAction = () =>
+  /* no change to crossfilter until a change fires */
+  ({ type: "graph brush start" });
+
+const _graphBrushWithinRectAction = (type, embName, brushCoords) => async (
+  dispatch,
+  getState
+) => {
+  const { obsCrossfilter: prevObsCrossfilter } = getState();
+
+  const selection = { mode: "within-rect", ...brushCoords };
+  const obsCrossfilter = await prevObsCrossfilter.select(
+    "emb",
+    embName,
+    selection
+  );
+
+  dispatch({
+    type,
+    obsCrossfilter,
+    brushCoords,
+  });
+};
+
+const _graphAllAction = (type, embName) => async (dispatch, getState) => {
+  const { obsCrossfilter: prevObsCrossfilter } = getState();
+
+  const obsCrossfilter = await prevObsCrossfilter.select("emb", embName, {
+    mode: "all",
+  });
+
+  dispatch({
+    type,
+    obsCrossfilter,
+  });
+};
+
+export const graphBrushChangeAction = (embName, brushCoords) =>
+  _graphBrushWithinRectAction("graph brush change", embName, brushCoords);
+
+export const graphBrushEndAction = (embName, brushCoords) =>
+  _graphBrushWithinRectAction("graph brush end", embName, brushCoords);
+
+export const graphBrushCancelAction = (embName) =>
+  _graphAllAction("graph brush cancel", embName);
+export const graphBrushDeselectAction = (embName) =>
+  _graphAllAction("graph brush deselect", embName);
+
+export const graphLassoStartAction = () =>
+  /* no change to crossfilter until a change fires */
+  ({ type: "graph lasso start" });
+
+export const graphLassoCancelAction = (embName) =>
+  _graphAllAction("graph lasso cancel", embName);
+
+export const graphLassoDeselectAction = (embName) =>
+  _graphAllAction("graph lasso cancel", embName);
+
+export const graphLassoEndAction = (embName, polygon) => async (
+  dispatch,
+  getState
+) => {
+  const { obsCrossfilter: prevObsCrossfilter } = getState();
+
+  const selection = {
+    mode: "within-polygon",
+    polygon,
+  };
+  const obsCrossfilter = await prevObsCrossfilter.select(
+    "emb",
+    embName,
+    selection
+  );
+
+  dispatch({
+    type: "graph lasso end",
+    obsCrossfilter,
+    polygon,
+  });
+};
+
+/*
+Differential expression set selection
+*/
+export const setCellSetFromSelection = (cellSetId) => (dispatch, getState) => {
+  const { obsCrossfilter } = getState();
+  const selected = obsCrossfilter.allSelectedLabels();
+
+  dispatch({
+    type: `store current cell selection as differential set ${cellSetId}`,
+    data: selected.length > 0 ? selected : null,
+  });
+};

client/src/actions/viewStack.js

@@ -0,0 +1,76 @@
+/*
+The following actions manage the view stack for annoMatrix.
+
+Conventions used and assumed elsewhere in the code base:
+  * there will be zero or one clip view, and it will be the TOP view always.
+  * there will be zero or more subset views
+
+In other words, in our current use, we do not stack multiple clip views but we do
+stack multiple subsets.
+
+If these conventions change, code elsewhere (eg. menubar/clip.js) will need to
+change as well.
+*/
+import { AnnoMatrixObsCrossfilter } from "../annoMatrix";
+import {
+  _clipAnnoMatrix,
+  _userSubsetAnnoMatrix,
+  _userResetSubsetAnnoMatrix,
+} from "../util/stateManager/viewStackHelpers";
+
+export const clipAction = (min, max) => (dispatch, getState) => {
+  /*
+  apply a clip to the current annoMatrix.  By convention, the clip
+  view is ALWAYS the top view.
+  */
+  const { annoMatrix: prevAnnoMatrix } = getState();
+  const annoMatrix = _clipAnnoMatrix(prevAnnoMatrix, min, max);
+  const obsCrossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
+  dispatch({
+    type: "set clip quantiles",
+    clipQuantiles: { min, max },
+    annoMatrix,
+    obsCrossfilter,
+  });
+};
+
+export const subsetAction = () => (dispatch, getState) => {
+  /*
+  Subset the annoMatrix to the current crossfilter selection by pushing a 
+  subset view.
+
+  By convention, a clip view is ALWAYS the top view, so if present, pop
+  off and re-apply
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
+  const annoMatrix = _userSubsetAnnoMatrix(
+    prevAnnoMatrix,
+    prevObsCrossfilter.allSelectedMask()
+  );
+  const obsCrossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
+  dispatch({
+    type: "subset to selection",
+    annoMatrix,
+    obsCrossfilter,
+  });
+};
+
+export const resetSubsetAction = () => (dispatch, getState) => {
+  /*
+  Reset the annoMatrix to all data.  Because we may have multiple views
+  stacked, we pop them all.  By convention, any clip transformation will
+  be the top of the stack, and must be preserved.
+  */
+
+  const { annoMatrix: prevAnnoMatrix } = getState();
+  const annoMatrix = _userResetSubsetAnnoMatrix(prevAnnoMatrix);
+  const obsCrossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
+  dispatch({
+    type: "reset subset",
+    annoMatrix,
+    obsCrossfilter,
+  });
+};

client/src/annoMatrix/annoMatrix.js

@@ -0,0 +1,658 @@
+import {
+  Dataframe,
+  IdentityInt32Index,
+  dataframeMemo,
+} from "../util/dataframe";
+import {
+  _getColumnDimensionNames,
+  _getColumnSchema,
+  _schemaColumns,
+  _getWritableColumns,
+} from "./schema";
+import { indexEntireSchema } from "../util/stateManager/schemaHelpers";
+import { _whereCacheGet, _whereCacheMerge } from "./whereCache";
+import _shallowClone from "./clone";
+
+const _dataframeCache = dataframeMemo(128);
+
+export default class AnnoMatrix {
+  /*
+  Abstract base class for all AnnoMatrix objects.  This class provides a proxy
+  to the annotated matrix data authoritatively served by the server/back-end.
+
+  AnnoMatrix instances are immutable, meaning that their schema and dimensionality
+  will not change, and simple object equality can be used to detect structural
+  changes. The actual data is cached, and not guaranteed to be present -- any
+  request to access data must be resolved by a fetch() call, which is async, and
+  may involve a server round-trip.
+
+  Guarantees made by the immutabilty, ie, any of these can be detected by
+  simple annoMatrix compare:
+    * schema is the same, including all fields and columns
+    * dimensionality is the same (nObs, nVar)
+    * data mapping/transformation, such as clipping, are the same
+
+  AnnoMatrixes also "stack" like filters, allowing for the construction of
+  views which transform the data in some manner.
+
+  The bootstrap class is AnnoMatrixLoader, which is the caching server proxy, and
+  is bootstrapped with a API URL:
+    new AnnoMatirx(url, schema) -> annoMatrix
+
+  There are various "views", such as AnnoMatrixRowSubsetView, which provide
+  the same interface but with a transformed view of the server data.  Utilities in
+  viewCreators.js can be used to create these views:
+      clip(annoMatrix, min, max) -> annoMatrix
+      subset(annoMatrix, rowLabels) -> annoMatrix
+  etc.
+  */
+  static fields() {
+    /*
+    return the fields present in the AnnoMatrix instance.
+    */
+    return ["obs", "var", "emb", "X"];
+  }
+
+  constructor(schema, nObs, nVar, rowIndex = null) {
+    /*
+    Private constructor - this is an abstract base class.  Do not use.
+    */
+
+    /*
+    Public instance fields:
+      * schema - the matrix schema.  IMPORTANT: always the entire schema, for the
+        base (unfiltered, unclipped, unsubset) annotated matrix, as the server
+        presents it.
+      * nObs, nVar - size of each dimension.  These will accurately reflect the
+        size of the current annoMatrix view.  For example, if you subset the view,
+        the nObs will be smaller.
+      * rowIndex - a rowIndex shared by all data on this view (ie, the list of cells).
+        The row index labels are as defined by the base dataset from the server.
+      * isView - true if this is a view, false if not.
+      * viewOf - pointer to parent annomatrix if a view, undefined/null if not a view.
+      * userFlags - container for any additional state a user of this API wants to hang
+        off of an annoMatrix, and have propagated by the (shallow) cloning protocol.
+    */
+    this.schema = indexEntireSchema(schema);
+    this.nObs = nObs;
+    this.nVar = nVar;
+    this.rowIndex = rowIndex || new IdentityInt32Index(nObs);
+    this.isView = false;
+    this.viewOf = undefined;
+    this.userFlags = {};
+
+    /*
+		Private instance variables.
+
+		These are caches - lazily loaded. The only guarantee is that if they
+		are loaded, they will conform to the schema & dimensionality constraints.
+
+		Do NOT use directly - instead, use the fetch() and preload() API.
+		*/
+    this._cache = {
+      obs: Dataframe.empty(this.rowIndex),
+      var: Dataframe.empty(this.rowIndex),
+      emb: Dataframe.empty(this.rowIndex),
+      X: Dataframe.empty(this.rowIndex),
+    };
+    this._pendingLoad = {
+      obs: {},
+      var: {},
+      emb: {},
+      X: {},
+    };
+    this._whereCache = {};
+    this._gcInfo = new Map();
+  }
+
+  /**
+   ** Schema helper/accessors
+   **/
+  getMatrixColumns(field) {
+    /*
+    Return array of column names in the field.  ONLY supported on the
+    obs, var and emb fields.  X currently unimplemented and will throw.
+
+    For exmaple:
+
+      annoMatrix.getMatrixColumns("obs") -> ["louvain", "n_genes"]
+    */
+    return _schemaColumns(this.schema, field);
+  }
+
+  // eslint-disable-next-line class-methods-use-this -- need to be able to call this on instances
+  getMatrixFields() {
+    /*
+    Return array of fields in this annoMatrix.  Currently hard-wired to
+    return:  ["X", "obs", "var", "emb"].
+
+    These are the fields from data may be requested.
+    */
+    return AnnoMatrix.fields();
+  }
+
+  getColumnSchema(field, col) {
+    /*
+    Return the schema for the field & column ,eg,
+
+      anonMatrix.getColumnSchema("obs", "n_genes") -> { type: "int32", name: "n_genes" }
+
+    This is identical to the information in the annoMatrix.schema
+    instance variable.
+    */
+    return _getColumnSchema(this.schema, field, col);
+  }
+
+  getColumnDimensions(field, col) {
+    /*
+    Return the dimensions on this field / column.  For most fields, which are 1D,
+    this just return the column name.  Multi-dimensional columns, such as embeddings,
+    will return >1 name.
+
+    Examples:
+
+      getColumnDimensions("obs", "louvain") -> ["louvain"]
+      getColumnDimensions("emb", "umap") -> ["umap_0", "umap_1"]
+
+    */
+    return _getColumnDimensionNames(this.schema, field, col);
+  }
+
+  /**
+   ** General utility methods
+   **/
+  base() {
+    /*
+    return the base of view, or `this` if not a view.
+    */
+    let annoMatrix = this;
+    while (annoMatrix.isView) annoMatrix = annoMatrix.viewOf;
+    return annoMatrix;
+  }
+
+  /**
+   ** Load / read interfaces
+   **/
+  fetch(field, q) {
+    /*
+		Return the given query on a single matrix field as a single dataframe.
+		Currently supports ONLY full column query.
+
+		Returns a Promise for the query result, which will resolve to a dataframe.
+
+		Field must be one of the matrix fields: 'obs', 'var', 'X', 'emb'.  Value
+		represents the underlying object upon which the query is occuring.
+
+		Query is one of:
+			* a string, representing a single column name from the field, eg,
+				"n_genes"
+			* an object, containing an "value" query (see below).
+			* an array, containing one or more of the above.
+
+		Columns may have more than one dimension, and all will be fetched
+		and returned together.  This is most commonly seen in an embedding,
+    which usually has two dimensions.
+
+		A value query allows for fetching based upon the value in another 
+		field/column, similar to a join.  Currently only supported on the var
+		dimension, allowing query of X columns by var value (eg, gene name)
+
+    The query filter	is a single value filter:
+			{ "field name": [
+				{name: "column name", values: [ list of values ]}
+			]}
+		One and only one value filter is allowed in a value query.
+
+		Examples:
+
+    1. Fetch the "n_genes" column the "obs":
+
+			const df = await fetch("obs", "n_genes")
+      console.log("Largest number of genes is: ", df.summarize().max);
+
+    2. Fetch two separate columns from obs.  Returns a single dataframe containing
+       the columns:
+
+			const df = await fetch("obs", ["n_genes", "louvain"])
+      console.log("Cell 0 has category: ", df.at(0, "louvain"));
+
+    3. Fetch an entire X (expression counts) column that has a var annotation
+       value "TYMP" in the var index.
+
+			fetch("X", { 
+				where: {field: "var", column: this.schema.annotations.var.index, value: "TYMP"}
+			})
+
+      In AnnData & Pandas DataFrame API, this is equivalent to:
+        adata.X[:, adata.var.index.get_loc("SUMO3")]
+
+		The value query is a recodification and subset of the server REST API 
+		value filter JSON.  Range queries and multiple filters are not currently
+		supported.
+
+		*/
+    return this._fetch(field, q);
+  }
+
+  prefetch(field, q) {
+    /*
+		Start a data fetch & cache fill.  Identical to fetch() except it does
+		not return a value.
+
+    Primary use is to being a cache load as early as is possible, reducing
+    overall component rendering latency.
+		*/
+    this._fetch(field, q);
+    return undefined;
+  }
+
+  /**
+   ** Save / mutate interfaces - manipulation of "writable" OBS annotations.
+   **
+   ** These are all present to support client-side creation of OBS annotations, aka
+   ** "user annotations".
+   **
+   ** They implement common manipulations to the AnnoMatrix, maintaining the
+   ** norma guarantees around correctness of public API, eg,
+   **   - schema will be correct, including the "writable" attribute
+   **   - fetch() will return the latest data, even from views
+   **   - immutability guranteeds
+   **
+   ** As most of these interfaces mutate the annoMatrix, they return a new
+   ** annoMatrix
+   **
+   ** The actual implementation is in the sub-classes, which MUST override these.
+   **/
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  addObsAnnoCategory(col, category) {
+    /*
+    Add a new category value (aka "label") to a writable obs column, and return the new AnnoMatrix.
+    Typical use is to add a new user-created label to a user-created obs categorical
+    annotation.
+
+    Will throw column does not exist or is not writable.
+
+    Example:
+
+      addObsAnnoCategory("my cell type", "left toenail") -> AnnoMatrix
+
+    */
+    _subclassResponsibility();
+  }
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  async removeObsAnnoCategory(col, category, unassignedCategory) {
+    /*
+    Remove a category value from an obs column, reassign any obs having that value
+    to the 'unassignedCategory' value, and return a promise for a new AnnoMatrix.
+    Typical use is to remove a user-created label from a user-created obs categorical
+    annotation.
+
+    Will throw column does not exist or is not writable.
+
+    An `unassignedCategory` value must be provided, for assignment to any obs/cells
+    that had the now-delete category label as their value.
+
+    Example:
+      await removeObsAnnoCategory("my-tissue-type", "right earlobe", "unassigned") -> AnnoMatrix
+
+    NOTE: method is async as it may need to fetch data to provide the reassignment.
+    */
+    _subclassResponsibility();
+  }
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  dropObsColumn(col) {
+    /*
+    Drop an entire writable column, eg a user-created obs annotation.  Typical use
+    is to provide the "Delete Category" implementation.  Returns the new AnnoMatrix.
+    Will throw if not a writable annotation.
+
+    Will throw column does not exist or is not writable.
+
+    Example:
+
+      dropObsColumn("old annotations") ->  AnnoMatrix
+    */
+    _subclassResponsibility();
+  }
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  addObsColumn(colSchema, Ctor, value) {
+    /*
+    Add a new writable OBS annotation column, with the caller-specified schema, initial value
+    type and value.
+
+    Value may be any one of:
+      * an array of values
+      * a primitive type, including null or undefined.
+    If an array, length must be the same as 'this.nObs', and constructor must equal 'Ctor'.
+    If a primitive, 'Ctor' will be used to create the initial value, which will be filled
+    with 'value'.
+
+    Throws if the name specified in 'colSchema' duplicates an existing obs column.
+
+    Returns a new AnnoMatrix.
+
+    Examples:
+
+      addObsColumn(
+        { name: "foo", type: "categorical", categories: "unassigned" },
+        Array,
+        "unassigned"
+      ) -> AnnoMatrix
+
+    */
+    _subclassResponsibility();
+  }
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  renameObsColumn(oldCol, newCol) {
+    /*
+    Rename the obs column 'oldCol' to have name 'newCol' and returns new AnnoMatrix.
+
+    Will throw column does not exist or is not writable, or if 'newCol' is not unique.
+
+    Example:
+
+      renameObsColumn('cell type', 'old cell type') -> AnnoMatrix.
+
+    */
+    _subclassResponsibility();
+  }
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  async setObsColumnValues(col, obsLabels, value) {
+    /*
+    Set all obs with label in array 'obsLabels' to have 'value'.  Typical use would be
+    to set a group of cells to have a label on a user-created categorical anntoation
+    (eg set all selected cells to have a label).
+
+    NOTE: async method, as it may need to fetch.
+
+    Will throw column does not exist or is not writable.
+
+    Example:
+      await setObsColmnValues("flavor", [383, 400], "tasty") -> AnnoMtarix
+
+    */
+    _subclassResponsibility();
+  }
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  async resetObsColumnValues(col, oldValue, newValue) {
+    /*
+    Set by value - all elements in the column with value 'oldValue' are set to 'newValue'.
+    Async method - returns a promise for a new AnnoMatrix.
+
+    Typical use would be to set all labels of one value to another.
+
+    Will throw column does not exist or is not writable.
+
+    Example:
+      await resetObsColumnValues("my notes", "good", "not-good") -> AnnoMatrix
+
+    */
+    _subclassResponsibility();
+  }
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  addEmbedding(colSchema) {
+    /*
+    Add a new obs embedding to the AnnoMatrix, with provided schema.
+    Returns a new annomatrix.
+
+    Typical use will be to add a re-embedding that the server has calculated.
+
+    Will throw if the column schema is invalid (eg, duplicate name).
+    */
+    _subclassResponsibility();
+  }
+
+  /**
+   ** Private interfaces below.
+   **/
+  _resolveCachedQueries(field, queries) {
+    return queries
+      .map((query) =>
+        _whereCacheGet(this._whereCache, this.schema, field, query).filter(
+          (cacheKey) =>
+            cacheKey !== undefined && this._cache[field].hasCol(cacheKey)
+        )
+      )
+      .flat();
+  }
+
+  async _fetch(field, q) {
+    if (!AnnoMatrix.fields().includes(field)) return undefined;
+    const queries = Array.isArray(q) ? q : [q];
+
+    /* find cached columns we need, and GC the rest */
+    const cachedColumns = this._resolveCachedQueries(field, queries);
+    this._gcFetchCleanup(field, cachedColumns);
+
+    /* find any query not already cached */
+    const uncachedQueries = queries.filter((query) =>
+      _whereCacheGet(this._whereCache, this.schema, field, query).some(
+        (cacheKey) =>
+          cacheKey === undefined || !this._cache[field].hasCol(cacheKey)
+      )
+    );
+
+    /* load uncached queries */
+    if (uncachedQueries.length > 0) {
+      await Promise.all(
+        uncachedQueries.map((query) =>
+          this._getPendingLoad(field, query, async (_field, _query) => {
+            /* fetch, then index.  _doLoad is subclass interface */
+            const [whereCacheUpdate, df] = await this._doLoad(_field, _query);
+            this._cache[_field] = this._cache[_field].withColsFrom(df);
+            this._whereCache = _whereCacheMerge(
+              this._whereCache,
+              whereCacheUpdate
+            );
+          })
+        )
+      );
+    }
+
+    /* everything we need is in the cache, so just cherry-pick requested columns */
+    const requestedCacheKeys = this._resolveCachedQueries(field, queries);
+    const response = _dataframeCache(
+      this._cache[field].subset(null, requestedCacheKeys)
+    );
+    this._gcUpdateStats(field, response);
+    return response;
+  }
+
+  async _getPendingLoad(field, query, fetchFn) {
+    /*
+    Given a query on a field, ensure that we only have a single outstanding
+    fetch at any given time.  If multiple requests occur while a fetch is
+    outstanding, just wait for the original.
+
+    This is implemented by returning a promise that will await the singular
+    fetch promise.
+    */
+    const key = _queryCacheKey(field, query);
+    if (!this._pendingLoad[field][key]) {
+      this._pendingLoad[field][key] = fetchFn(field, query);
+      try {
+        await this._pendingLoad[field][key];
+      } finally {
+        delete this._pendingLoad[field][key];
+      }
+    }
+    return this._pendingLoad[field][key];
+  }
+
+  // eslint-disable-next-line class-methods-use-this -- make sure subclass implements
+  async _doLoad() {
+    _subclassResponsibility();
+  }
+
+  /**
+   ** Garbage collection of annomatrix cache to manage memory use.
+   **/
+
+  /*
+  These callbacks implement a GC policy for the cache.  Background:
+
+    * For the Loader (base) annomatrix, re-filling the cache is expensive as
+      it requires an HTTP fetch.
+    * user-defined / writable columns must not be GC'ed as they may be
+      still pending a save/commit.
+    * For views, cost is less and (roughly) proportional with nObs
+    * obs, var and emb do not grow without bounds, and are needed constantly 
+      for rendering.  
+      a) There is no upside to GC'ing these in the base (loader)
+      b) The undo/redo cache can hold a large number in views, which is worht GC'ing
+    * X is often much larger than memory, and the UI allows add/del from
+      this.  Most of the GC potential is here in both the base and views.
+
+  Current policy:
+    * if in active use ("hot") do not GC obs, var or emb.
+    * never, ever GC writable obs columns
+    * For base/loader set a numeric limit on maximum X column count
+    * For views, apply a fixed limit to the number of columns cached in any field.
+      Limit will be lower if not hot.
+
+  To be effective, the GC callback needs to be invoked from the undo/redo code,
+  as much of the cache is pinned by that data structure.
+  */
+  _gcField(field, isHot, pinnedColumns) {
+    const maxColumns = isHot ? 256 : 10; // maybe to aggessive?
+
+    const cache = this._cache[field];
+    if (cache.colIndex.size() < maxColumns) return; // trivial rejection
+
+    const candidates = cache.colIndex
+      .labels()
+      .filter((col) => !pinnedColumns.includes(col));
+
+    const excessCount = candidates.length + pinnedColumns.length - maxColumns;
+    if (excessCount > 0) {
+      const { _gcInfo } = this;
+      candidates.sort((a, b) => {
+        let atime = _gcInfo.get(_columnCacheKey(field, a));
+        if (atime === undefined) atime = 0;
+
+        let btime = _gcInfo.get(_columnCacheKey(field, b));
+        if (btime === undefined) btime = 0;
+
+        return atime - btime;
+      });
+
+      const toDrop = candidates.slice(0, excessCount);
+      // helpful debugging - please leave in place.
+      // console.log(
+      //   `GC: dropping from ${field} hot:${isHot}, columns [${toDrop.join(
+      //     ", "
+      //   )}]`
+      // );
+      this._cache[field] = toDrop.reduce(
+        (df, col) => df.dropCol(col),
+        this._cache[field]
+      );
+      toDrop.forEach((col) => _gcInfo.delete(_columnCacheKey(field, col)));
+    }
+  }
+
+  _gcFetchCleanup(field, pinnedColumns) {
+    /*
+    Called during data load/fetch.  By definition, this is 'hot', so we
+    only want to gc X.
+    */
+    if (field === "X") {
+      this._gcField(
+        field,
+        true,
+        pinnedColumns.concat(_getWritableColumns(this.schema, field))
+      );
+    }
+  }
+
+  _gc(hints) {
+    /*
+    Called from middleware, or elsewhere.  isHot is true if we are in the active store, 
+    or false if we are in some other context (eg, history state).
+    */
+    const { isHot } = hints;
+    const candidateFields = isHot ? ["X"] : ["X", "emb", "var", "obs"];
+    candidateFields.forEach((field) =>
+      this._gcField(field, isHot, _getWritableColumns(this.schema, field))
+    );
+  }
+
+  _gcUpdateStats(field, dataframe) {
+    /*
+    called each time a query is performed, allowing the gc to update any bookkeeping
+    information.  Currently, this is just a simple last-fetched timestamp, stored
+    in a Map.
+
+    Map objects preserve order of insertion. This is leveraged as a cheap way to
+    do LRU, by removing and re-inserting keys.  IMPORTANT: the cleanup code assumes
+    the map insertion order is least-recently-used first.
+    */
+    const cols = dataframe.colIndex.labels();
+    const { _gcInfo } = this;
+    const now = Date.now();
+    cols.forEach((c) => {
+      // gcInfo.delete(c);
+      _gcInfo.set(_columnCacheKey(field, c), now);
+    });
+  }
+
+  /**
+  Cloning sublcass protocol - we rely in cloning to preserve immutable
+  symantics while not causing races or other side effects in internal
+  cache management.
+
+  Subclasses must override _cloneDeeper() if they have state which requires 
+  something other than a shallow copy.  Overrides MUST call super()._cloneDeepr(), 
+  and return its result (after any required modification).  _cloneDeeper()
+  will be called on the OLD object, with the NEW object as an argument.
+
+  Do not override _clone();
+  **/
+  _cloneDeeper(clone) {
+    clone._cache = _shallowClone(this._cache);
+    clone._gcInfo = new Map();
+    clone._pendingLoad = {
+      obs: {},
+      var: {},
+      emb: {},
+      X: {},
+    };
+    return clone;
+  }
+
+  _clone() {
+    const clone = _shallowClone(this);
+    this._cloneDeeper(clone);
+    Object.seal(clone);
+    return clone;
+  }
+}
+
+/*
+private utility functions below
+*/
+
+function _queryCacheKey(field, query) {
+  if (typeof query === "object") {
+    const { field: queryField, column: queryColumn, value: queryValue } = query;
+    return `${field}/${queryField}/${queryColumn}/${queryValue}`;
+  }
+  return `${field}/${query}`;
+}
+
+function _columnCacheKey(field, column) {
+  return `${field}/${column}`;
+}
+
+function _subclassResponsibility() {
+  /* protect against bugs in subclass */
+  throw new Error("subclass failed to implement required method");
+}

client/src/annoMatrix/clone.js

@@ -0,0 +1,6 @@
+/*
+Shallow clone an object, correctly handling prototype
+*/
+export default function _shallowClone(orig) {
+  return Object.assign(Object.create(Object.getPrototypeOf(orig)), orig);
+}

client/src/annoMatrix/crossfilter.js

@@ -0,0 +1,268 @@
+/*
+Row crossfilter proxy for an AnnoMatrix.  This wraps Crossfilter,
+providing a number of services, and ensuring that the crossfilter and
+AnnoMatrix stay in sync:
+	- on-demand index creation as data is loaded
+	- transparently mapping between queries and crossfilter index names.
+  - for mutation of the matrix by user annotations, maintain synchronization
+    between Crossfilter and AnnoMatrix.
+*/
+import Crossfilter from "../util/typedCrossfilter";
+import { _getColumnSchema } from "./schema";
+
+function _dimensionNameFromDf(field, df) {
+  const colNames = df.colIndex.labels();
+  return _dimensionName(field, colNames);
+}
+
+function _dimensionName(field, colNames) {
+  if (!Array.isArray(colNames)) return `${field}/${colNames}`;
+  return `${field}/${colNames.join(":")}`;
+}
+
+export default class AnnoMatrixObsCrossfilter {
+  constructor(annoMatrix, _obsCrossfilter = null) {
+    this.annoMatrix = annoMatrix;
+    this.obsCrossfilter =
+      _obsCrossfilter || new Crossfilter(annoMatrix._cache.obs);
+    this.obsCrossfilter = this.obsCrossfilter.setData(annoMatrix._cache.obs);
+  }
+
+  size() {
+    return this.obsCrossfilter.size();
+  }
+
+  /**
+  Managing the associated annoMatrix.  These wrappers are necessary to 
+  make coordinated changes to BOTH the crossfilter and annoMatrix, and
+  ensure that all state stays synchronized.
+
+  See API documentation in annoMatrix.js.
+  **/
+  addObsColumn(colSchema, Ctor, value) {
+    const annoMatrix = this.annoMatrix.addObsColumn(colSchema, Ctor, value);
+    const obsCrossfilter = this.obsCrossfilter.setData(annoMatrix._cache.obs);
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  dropObsColumn(col) {
+    const annoMatrix = this.annoMatrix.dropObsColumn(col);
+    let { obsCrossfilter } = this;
+    const dimName = _dimensionName("obs", col);
+    if (obsCrossfilter.hasDimension(dimName)) {
+      obsCrossfilter = obsCrossfilter.delDimension(dimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  renameObsColumn(oldCol, newCol) {
+    const annoMatrix = this.annoMatrix.renameObsColumn(oldCol, newCol);
+    const oldDimName = _dimensionName("obs", oldCol);
+    const newDimName = _dimensionName("obs", newCol);
+    let { obsCrossfilter } = this;
+    if (obsCrossfilter.hasDimension(oldDimName)) {
+      obsCrossfilter = obsCrossfilter.renameDimension(oldDimName, newDimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  addObsAnnoCategory(col, category) {
+    const annoMatrix = this.annoMatrix.addObsAnnoCategory(col, category);
+    const dimName = _dimensionName("obs", col);
+    let { obsCrossfilter } = this;
+    if (obsCrossfilter.hasDimension(dimName)) {
+      obsCrossfilter = obsCrossfilter.delDimension(dimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  async removeObsAnnoCategory(col, category, unassignedCategory) {
+    const annoMatrix = await this.annoMatrix.removeObsAnnoCategory(
+      col,
+      category,
+      unassignedCategory
+    );
+    const dimName = _dimensionName("obs", col);
+    let { obsCrossfilter } = this;
+    if (obsCrossfilter.hasDimension(dimName)) {
+      obsCrossfilter = obsCrossfilter.delDimension(dimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  async setObsColumnValues(col, rowLabels, value) {
+    const annoMatrix = await this.annoMatrix.setObsColumnValues(
+      col,
+      rowLabels,
+      value
+    );
+    const dimName = _dimensionName("obs", col);
+    let { obsCrossfilter } = this;
+    if (obsCrossfilter.hasDimension(dimName)) {
+      obsCrossfilter = obsCrossfilter.delDimension(dimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  async resetObsColumnValues(col, oldValue, newValue) {
+    const annoMatrix = await this.annoMatrix.resetObsColumnValues(
+      col,
+      oldValue,
+      newValue
+    );
+    const dimName = _dimensionName("obs", col);
+    let { obsCrossfilter } = this;
+    if (obsCrossfilter.hasDimension(dimName)) {
+      obsCrossfilter = obsCrossfilter.delDimension(dimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  addEmbedding(colSchema) {
+    const annoMatrix = this.annoMatrix.addEmbedding(colSchema);
+    return new AnnoMatrixObsCrossfilter(annoMatrix, this.obsCrossfilter);
+  }
+
+  /**
+  Selection state - API is identical to ImmutableTypedCrossfilter, as these
+  are just wrappers to lazy create indices.
+  **/
+
+  async select(field, query, spec) {
+    const { annoMatrix } = this;
+    let { obsCrossfilter } = this;
+
+    if (!annoMatrix?._cache?.[field]) {
+      throw new Error("Unknown field name");
+    }
+    if (field === "var") {
+      throw new Error("unable to obsSelect upon the var dimension");
+    }
+
+    // grab the data, so we can grab the index.
+    const df = await annoMatrix.fetch(field, query);
+
+    const dimName = _dimensionNameFromDf(field, df);
+    if (!obsCrossfilter.hasDimension(dimName)) {
+      // lazy index generation - add dimension when first used
+      obsCrossfilter = this._addObsCrossfilterDimension(
+        annoMatrix,
+        obsCrossfilter,
+        field,
+        df
+      );
+    }
+
+    // select
+    obsCrossfilter = obsCrossfilter.select(dimName, spec);
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  selectAll() {
+    /*
+		Select all on any dimension in this field.
+		*/
+    const { annoMatrix } = this;
+    const currentDims = this.obsCrossfilter.dimensionNames();
+    const obsCrossfilter = currentDims.reduce((xfltr, dim) => {
+      return xfltr.select(dim, { mode: "all" });
+    }, this.obsCrossfilter);
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  countSelected() {
+    /* if no data yet indexed in the crossfilter, just say everything is selected */
+    if (this.obsCrossfilter.size() === 0) return this.annoMatrix.nObs;
+    return this.obsCrossfilter.countSelected();
+  }
+
+  allSelectedMask() {
+    /* if no data yet indexed in the crossfilter, just say everything is selected */
+    if (
+      this.obsCrossfilter.size() === 0 ||
+      this.obsCrossfilter.dimensionNames().length === 0
+    ) {
+      /* fake the mask */
+      return new Uint8Array(this.annoMatrix.nObs).fill(1);
+    }
+    return this.obsCrossfilter.allSelectedMask();
+  }
+
+  allSelectedLabels() {
+    /* if no data yet indexed in the crossfilter, just say everything is selected */
+    if (
+      this.obsCrossfilter.size() === 0 ||
+      this.obsCrossfilter.dimensionNames().length === 0
+    ) {
+      return this.annoMatrix.rowIndex.labels();
+    }
+
+    const mask = this.obsCrossfilter.allSelectedMask();
+    const index = this.annoMatrix.rowIndex.isubsetMask(mask);
+    return index.labels();
+  }
+
+  fillByIsSelected(array, selectedValue, deselectedValue) {
+    /* if no data yet indexed in the crossfilter, just say everything is selected */
+    if (
+      this.obsCrossfilter.size() === 0 ||
+      this.obsCrossfilter.dimensionNames().length === 0
+    ) {
+      return array.fill(selectedValue);
+    }
+    return this.obsCrossfilter.fillByIsSelected(
+      array,
+      selectedValue,
+      deselectedValue
+    );
+  }
+
+  /**
+   ** Private below
+   **/
+
+  _addObsCrossfilterDimension(annoMatrix, obsCrossfilter, field, df) {
+    if (field === "var") return obsCrossfilter;
+    const dimName = _dimensionNameFromDf(field, df);
+    const dimParams = this._getObsDimensionParams(field, df);
+    obsCrossfilter = obsCrossfilter.setData(annoMatrix._cache.obs);
+    obsCrossfilter = obsCrossfilter.addDimension(dimName, ...dimParams);
+    return obsCrossfilter;
+  }
+
+  _getColumnBaseType(field, col) {
+    /* Look up the primitive type for this field/col */
+    const colSchema = _getColumnSchema(this.annoMatrix.schema, field, col);
+    return colSchema.type;
+  }
+
+  _getObsDimensionParams(field, df) {
+    /* return the crossfilter dimensiontype type and params for this field/dataframe */
+
+    if (field === "emb") {
+      /* assumed to be 2D */
+      return ["spatial", df.icol(0).asArray(), df.icol(1).asArray()];
+    }
+
+    /* assumed to be 1D */
+    const col = df.icol(0);
+    const colName = df.colIndex.getLabel(0);
+    const type = this._getColumnBaseType(field, colName);
+    if (type === "string" || type === "categorical" || type === "boolean") {
+      return ["enum", col.asArray()];
+    }
+    if (type === "int32") {
+      return ["scalar", col.asArray(), Int32Array];
+    }
+    if (type === "float32") {
+      return ["scalar", col.asArray(), Float32Array];
+    }
+    // Currently not supporting boolean and categorical types.
+    console.error(
+      `Warning - unknown metadata schema (${type}) for field ${field} ${colName}.`
+    );
+    // skip it - we don't know what to do with this type
+
+    return undefined;
+  }
+}

client/src/annoMatrix/fetchHelpers.js

@@ -0,0 +1,27 @@
+export { doBinaryRequest } from "../util/actionHelpers";
+
+/* double URI encode - needed for query-param filters */
+export function _dubEncURIComp(s) {
+  return encodeURIComponent(encodeURIComponent(s));
+}
+
+/* currently unused, consider deleting */
+export function _fetchResult(promise) {
+  let _status = "pending";
+  const res = promise.then(
+    (r) => {
+      _status = "success";
+      return r;
+    },
+    (e) => {
+      _status = "error";
+      throw e;
+    }
+  );
+
+  res.status = () => {
+    return _status;
+  };
+
+  return res;
+}

client/src/annoMatrix/index.js

@@ -0,0 +1,15 @@
+/*
+AnnoMatrix -- Annotated Matrix exported interface
+
+Public API is defined in:
+
+	annoMatrix.js
+	viewCreators.js
+	crossfilter.js
+
+*/
+
+export { default as AnnoMatrixLoader } from "./loader";
+export * from "./viewCreators";
+export { default as AnnoMatrixObsCrossfilter } from "./crossfilter";
+export { default as gcMiddleware } from "./middleware";

client/src/annoMatrix/loader.js

@@ -0,0 +1,307 @@
+import { doBinaryRequest, _dubEncURIComp } from "./fetchHelpers";
+import { matrixFBSToDataframe } from "../util/stateManager/matrix";
+import { _getColumnSchema, _normalizeCategoricalSchema } from "./schema";
+import {
+  addObsAnnoColumn,
+  removeObsAnnoColumn,
+  addObsAnnoCategory,
+  removeObsAnnoCategory,
+  addObsLayout,
+} from "../util/stateManager/schemaHelpers";
+import { isArrayOrTypedArray } from "../util/typeHelpers";
+import { _whereCacheCreate } from "./whereCache";
+import AnnoMatrix from "./annoMatrix";
+import PromiseLimit from "../util/promiseLimit";
+
+const promiseThrottle = new PromiseLimit(5);
+
+export default class AnnoMatrixLoader extends AnnoMatrix {
+  /*
+  AnnoMatrix implementation which proxies to HTTP server using the CXG REST API.
+  Used as the base (non-view) instance.
+
+  Public API is same as AnnoMatrix class (refer there for API description),
+  with the addition of the constructor which bootstraps:
+
+    new AnnoMatrixLoader(serverBaseURL, schema) -> instance
+
+  */
+  constructor(baseURL, schema) {
+    const { nObs, nVar } = schema.dataframe;
+    super(schema, nObs, nVar);
+
+    if (baseURL[baseURL.length - 1] !== "/") {
+      // must have trailing slash
+      baseURL += "/";
+    }
+    this.baseURL = baseURL;
+    Object.seal(this);
+  }
+
+  /**
+   ** Public.  API described in base class.
+   **/
+  addObsAnnoCategory(col, category) {
+    /*
+    Add a new category (aka label) to the schema for an obs column.
+    */
+    const colSchema = _getColumnSchema(this.schema, "obs", col);
+    _writableCategoryTypeCheck(colSchema); // throws on error
+
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.schema = addObsAnnoCategory(this.schema, col, category);
+    return newAnnoMatrix;
+  }
+
+  async removeObsAnnoCategory(col, category, unassignedCategory) {
+    /*
+    Remove a single "category" (aka "label") from the data & schema of an obs column.
+    */
+    const colSchema = _getColumnSchema(this.schema, "obs", col);
+    _writableCategoryTypeCheck(colSchema); // throws on error
+
+    const newAnnoMatrix = await this.resetObsColumnValues(
+      col,
+      category,
+      unassignedCategory
+    );
+    newAnnoMatrix.schema = removeObsAnnoCategory(
+      newAnnoMatrix.schema,
+      col,
+      category
+    );
+    return newAnnoMatrix;
+  }
+
+  dropObsColumn(col) {
+    /*
+		drop column from field
+		*/
+    const colSchema = _getColumnSchema(this.schema, "obs", col);
+    _writableCheck(colSchema); // throws on error
+
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix._cache.obs = this._cache.obs.dropCol(col);
+    newAnnoMatrix.schema = removeObsAnnoColumn(this.schema, col);
+    return newAnnoMatrix;
+  }
+
+  addObsColumn(colSchema, Ctor, value) {
+    /*
+		add a column to field, initializing with value.  Value may 
+    be one of:
+      * an array of values
+      * a primitive type, including null or undefined.
+    If an array, it must be of same size as nObs and same type as Ctor
+		*/
+    colSchema.writable = true;
+    const colName = colSchema.name;
+    if (
+      _getColumnSchema(this.schema, "obs", colName) ||
+      this._cache.obs.hasCol(colName)
+    ) {
+      throw new Error("column already exists");
+    }
+
+    const newAnnoMatrix = this._clone();
+    let data;
+    if (isArrayOrTypedArray(value)) {
+      if (value.constructor !== Ctor)
+        throw new Error("Mismatched value array type");
+      if (value.length !== this.nObs)
+        throw new Error("Value array has incorrect length");
+      data = value.slice();
+    } else {
+      data = new Ctor(this.nObs).fill(value);
+    }
+    newAnnoMatrix._cache.obs = this._cache.obs.withCol(colName, data);
+    _normalizeCategoricalSchema(
+      colSchema,
+      newAnnoMatrix._cache.obs.col(colName)
+    );
+    newAnnoMatrix.schema = addObsAnnoColumn(this.schema, colName, colSchema);
+    return newAnnoMatrix;
+  }
+
+  renameObsColumn(oldCol, newCol) {
+    /*
+    Rename the obs oldColName to newColName.  oldCol must be writable.
+    */
+    const oldColSchema = _getColumnSchema(this.schema, "obs", oldCol);
+    _writableCheck(oldColSchema); // throws on error
+
+    const value = this._cache.obs.hasCol(oldCol)
+      ? this._cache.obs.col(oldCol).asArray()
+      : undefined;
+    return this.dropObsColumn(oldCol).addObsColumn(
+      {
+        ...oldColSchema,
+        name: newCol,
+      },
+      value.constructor,
+      value
+    );
+  }
+
+  async setObsColumnValues(col, rowLabels, value) {
+    /*
+		Set all rows identified by rowLabels to value.
+		*/
+    const colSchema = _getColumnSchema(this.schema, "obs", col);
+    _writableCategoryTypeCheck(colSchema); // throws on error
+
+    // ensure that we have the data in cache before we manipulate it
+    await this.fetch("obs", col);
+    if (!this._cache.obs.hasCol(col))
+      throw new Error("Internal error - user annotation data missing");
+
+    const rowIndices = this.rowIndex.getOffsets(rowLabels);
+    const data = this._cache.obs.col(col).asArray().slice();
+    for (let i = 0, len = rowIndices.length; i < len; i += 1) {
+      const idx = rowIndices[i];
+      if (idx === undefined) throw new Error("Unknown row label");
+      data[idx] = value;
+    }
+
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix._cache.obs = this._cache.obs.replaceColData(col, data);
+    const { categories } = colSchema;
+    if (!categories?.includes(value)) {
+      newAnnoMatrix.schema = addObsAnnoCategory(this.schema, col, value);
+    }
+    return newAnnoMatrix;
+  }
+
+  async resetObsColumnValues(col, oldValue, newValue) {
+    /*
+    Set all rows with value 'oldValue' to 'newValue'.
+    */
+    const colSchema = _getColumnSchema(this.schema, "obs", col);
+    _writableCategoryTypeCheck(colSchema); // throws on error
+
+    if (!colSchema.categories.includes(oldValue)) {
+      throw new Error("unknown category");
+    }
+
+    // ensure that we have the data in cache before we manipulate it
+    await this.fetch("obs", col);
+    if (!this._cache.obs.hasCol(col))
+      throw new Error("Internal error - user annotation data missing");
+
+    const data = this._cache.obs.col(col).asArray().slice();
+    for (let i = 0, l = data.length; i < l; i += 1) {
+      if (data[i] === oldValue) data[i] = newValue;
+    }
+
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix._cache.obs = this._cache.obs.replaceColData(col, data);
+    const { categories } = colSchema;
+    if (!categories?.includes(newValue)) {
+      newAnnoMatrix.schema = addObsAnnoCategory(this.schema, col, newValue);
+    }
+    return newAnnoMatrix;
+  }
+
+  addEmbedding(colSchema) {
+    /*
+    add new layout to the obs embeddings
+    */
+    const { name: colName } = colSchema;
+    if (_getColumnSchema(this.schema, "emb", colName)) {
+      throw new Error("column already exists");
+    }
+
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.schema = addObsLayout(this.schema, colSchema);
+    return newAnnoMatrix;
+  }
+
+  /**
+   ** Private below
+   **/
+  async _doLoad(field, query) {
+    /*
+    _doLoad - evaluates the query against the field. Returns:
+      * whereCache update: column query map mapping the query to the column labels
+      * Dataframe containing the new colums (one per dimension)
+    */
+    let urlQuery;
+    let urlBase;
+    let priority = 10; // default fetch priority
+
+    switch (field) {
+      case "obs":
+      case "var": {
+        urlBase = `${this.baseURL}annotations/${field}`;
+        urlQuery = _encodeQuery("annotation-name", query);
+        break;
+      }
+      case "X": {
+        urlBase = `${this.baseURL}data/var`;
+        urlQuery = _encodeQuery(undefined, query);
+        break;
+      }
+      case "emb": {
+        urlBase = `${this.baseURL}layout/obs`;
+        urlQuery = _encodeQuery("layout-name", query);
+        priority = 0; // high prio load for embeddings
+        break;
+      }
+      default:
+        throw new Error("Unknown field name");
+    }
+
+    const url = `${urlBase}?${urlQuery}`;
+    const buffer = await promiseThrottle.priorityAdd(
+      priority,
+      doBinaryRequest,
+      url
+    );
+    const result = matrixFBSToDataframe(buffer);
+    if (!result || result.isEmpty()) throw Error("Unknown field/col");
+
+    const whereCacheUpdate = _whereCacheCreate(
+      field,
+      query,
+      result.colIndex.labels()
+    );
+
+    if (field === "obs") {
+      /* cough, cough - see comment on method */
+      _normalizeCategoricalSchema(
+        this.schema.annotations.obsByName[query],
+        result.col(query)
+      );
+    }
+
+    return [whereCacheUpdate, result];
+  }
+}
+
+/*
+Utility functions below
+*/
+
+function _encodeQuery(colKey, q) {
+  if (typeof q === "object") {
+    const { field: queryField, column: queryColumn, value: queryValue } = q;
+    return `${_dubEncURIComp(queryField)}:${_dubEncURIComp(
+      queryColumn
+    )}=${_dubEncURIComp(queryValue)}`;
+  }
+  if (!colKey) throw new Error("Unsupported query by name");
+  return `${colKey}=${encodeURIComponent(q)}`;
+}
+
+function _writableCheck(colSchema) {
+  if (!colSchema?.writable) {
+    throw new Error("Unknown or readonly obs column");
+  }
+}
+
+function _writableCategoryTypeCheck(colSchema) {
+  _writableCheck(colSchema);
+  if (colSchema.type !== "categorical") {
+    throw new Error("column must be categorical");
+  }
+}

client/src/annoMatrix/middleware.js

@@ -0,0 +1,66 @@
+/*
+Garbage collection / cache management support
+
+Middleware that knows how to pull annoMatrix from the undoable state,
+and pass it along to the AnnoMatrix class for possible cache GC.
+
+Private interface.
+
+Future work item: this middleware knows internal details of both the
+Undoable metareducer and the AnnoMatrix private API.  It would be helpful
+to make the Undoable interface better factored.
+*/
+
+const annoMatrixGC = (store) => (next) => (action) => {
+  if (_itIsTimeForGC()) {
+    _doGC(store);
+  }
+  return next(action);
+};
+
+let lastGCTime = 0;
+const InterGCDelayMS = 30 * 1000; // 30 seconds
+function _itIsTimeForGC() {
+  /*
+  we don't want to run GC on every dispatch, so throttle it a bit.
+
+  Runs every InterGCDelay period
+  */
+  const now = Date.now();
+  if (now - lastGCTime > InterGCDelayMS) {
+    lastGCTime = now;
+    return true;
+  }
+  return false;
+}
+
+function _doGC(store) {
+  const state = store.getState();
+
+  // these should probably be a function imported from undoable.js, etc, as
+  // they have overly intimiate knowledge of our reducers.
+  const undoablePast = state["@@undoable/past"];
+  const undoableFuture = state["@@undoable/future"];
+  const undoableStack = undoablePast
+    .concat(undoableFuture)
+    .flatMap((snapshot) =>
+      snapshot.filter((v) => v[0] === "annoMatrix").map((v) => v[1])
+    );
+  const currentAnnoMatrix = state.annoMatrix;
+
+  /*
+  We want to identify those matrixes currently "hot", ie, linked from the current annoMatrix,
+  as our current gc algo is more aggressive with those not hot.
+  */
+  const allAnnoMatrices = new Map(
+    undoableStack.map((m) => [m, { isHot: false }])
+  );
+  let am = currentAnnoMatrix;
+  while (am) {
+    allAnnoMatrices.set(am, { isHot: true });
+    am = am.viewOf;
+  }
+  allAnnoMatrices.forEach((hints, annoMatrix) => annoMatrix._gc(hints));
+}
+
+export default annoMatrixGC;

client/src/annoMatrix/schema.js

@@ -0,0 +1,88 @@
+/*
+Private helper functions related to schema
+*/
+import catLabelSort from "../util/catLabelSort";
+import { unassignedCategoryLabel } from "../globals";
+
+export function _getColumnSchema(schema, field, col) {
+  /* look up the column definition */
+  switch (field) {
+    case "obs":
+      if (typeof col === "object")
+        throw new Error("unable to get column schema by query");
+      return schema.annotations.obsByName[col];
+    case "var":
+      if (typeof col === "object")
+        throw new Error("unable to get column schema by query");
+      return schema.annotations.varByName[col];
+    case "emb":
+      if (typeof col === "object")
+        throw new Error("unable to get column schema by query");
+      return schema.layout.obsByName[col];
+    case "X":
+      return schema.dataframe;
+    default:
+      throw new Error(`unknown field name: ${field}`);
+  }
+}
+
+export function _getColumnDimensionNames(schema, field, col) {
+  /*
+		field/col may be an alias for multiple columns. Currently used to map ND 
+		values to 1D dataframe columns for embeddings/layout. Signfied by the presence
+		of the "dims" value in the schema.
+		*/
+  const colSchema = _getColumnSchema(schema, field, col);
+  if (!colSchema) {
+    return undefined;
+  }
+  return colSchema.dims || [col];
+}
+
+export function _schemaColumns(schema, field) {
+  switch (field) {
+    case "obs":
+      return Object.keys(schema.annotations.obsByName);
+    case "var":
+      return Object.keys(schema.annotations.varByName);
+    case "emb":
+      return Object.keys(schema.layout.obsByName);
+    default:
+      throw new Error(`unknown field name: ${field}`);
+  }
+}
+
+export function _getWritableColumns(schema, field) {
+  if (field !== "obs") return [];
+  return schema.annotations.obs.columns
+    .filter((v) => v.writable)
+    .map((v) => v.name);
+}
+
+export function _isContinuousType(schema) {
+  const { type } = schema;
+  return !(type === "string" || type === "boolean" || type === "categorical");
+}
+
+export function _normalizeCategoricalSchema(colSchema, col) {
+  const { type, writable } = colSchema;
+  if (
+    type === "string" ||
+    type === "boolean" ||
+    type === "categorical" ||
+    writable
+  ) {
+    const categorySet = new Set(
+      col.summarizeCategorical().categories.concat(colSchema.categories ?? [])
+    );
+    if (writable && !categorySet.has(unassignedCategoryLabel)) {
+      categorySet.add(unassignedCategoryLabel);
+    }
+    colSchema.categories = Array.from(categorySet);
+  }
+
+  if (colSchema.categories) {
+    colSchema.categories = catLabelSort(writable, colSchema.categories);
+  }
+  return colSchema;
+}

client/src/annoMatrix/viewCreators.js

@@ -0,0 +1,70 @@
+/*
+View creators.  These are helper functions which create new views from existing
+instances of AnnoMatrix, implementing common UI functions.
+*/
+
+import { AnnoMatrixRowSubsetView, AnnoMatrixClipView } from "./views";
+
+export function isubsetMask(annoMatrix, obsMask) {
+  /*
+		Subset annomatrix to contain the rows which have truish value in the mask.
+    Maks length must equal annoMatrix.nObs (row count).
+	*/
+  return isubset(annoMatrix, _maskToList(obsMask));
+}
+
+export function isubset(annoMatrix, obsOffsets) {
+  /*
+		Subset annomatrix to contain the positions contained in the obsOffsets array
+
+    Example:
+
+      isubset(annoMatrix, [0, 1]) -> annoMatrix with only the first two rows
+	*/
+  const obsIndex = annoMatrix.rowIndex.isubset(obsOffsets);
+  return new AnnoMatrixRowSubsetView(annoMatrix, obsIndex);
+}
+
+export function subset(annoMatrix, obsLabels) {
+  /*
+		subset based on labels
+  */
+  const obsIndex = annoMatrix.rowIndex.subset(obsLabels);
+  return new AnnoMatrixRowSubsetView(annoMatrix, obsIndex);
+}
+
+export function subsetByIndex(annoMatrix, obsIndex) {
+  /*
+  subset based upon the new obs index.
+  */
+  return new AnnoMatrixRowSubsetView(annoMatrix, obsIndex);
+}
+
+export function clip(annoMatrix, qmin, qmax) {
+  /*
+		Create a view that clips all continuous data to the [min, max] range.
+		The matrix shape does not change, but the continuous values outside the
+		specified range will become a NaN.
+		*/
+  return new AnnoMatrixClipView(annoMatrix, qmin, qmax);
+}
+
+/*
+Private utility functions below
+*/
+
+function _maskToList(mask) {
+  /* convert masks to lists - method wastes space, but is fast */
+  if (!mask) {
+    return null;
+  }
+  const list = new Int32Array(mask.length);
+  let elems = 0;
+  for (let i = 0, l = mask.length; i < l; i += 1) {
+    if (mask[i]) {
+      list[elems] = i;
+      elems += 1;
+    }
+  }
+  return list.subarray(0, elems);
+}

client/src/annoMatrix/views.js

@@ -0,0 +1,176 @@
+/* eslint-disable max-classes-per-file -- Classes are interrelated*/
+
+/*
+Views on the annomatrix.  all API here is defined in viewCreators.js and annoMatrix.js.
+*/
+import clip from "../util/clip";
+import AnnoMatrix from "./annoMatrix";
+import { _whereCacheCreate } from "./whereCache";
+import { _isContinuousType, _getColumnSchema } from "./schema";
+
+class AnnoMatrixView extends AnnoMatrix {
+  constructor(viewOf, rowIndex = null) {
+    const nObs = rowIndex ? rowIndex.size() : viewOf.nObs;
+    super(viewOf.schema, nObs, viewOf.nVar, rowIndex || viewOf.rowIndex);
+    this.viewOf = viewOf;
+    this.isView = true;
+  }
+
+  addObsAnnoCategory(col, category) {
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.viewOf = this.viewOf.addObsAnnoCategory(col, category);
+    newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema;
+    return newAnnoMatrix;
+  }
+
+  async removeObsAnnoCategory(col, category, unassignedCategory) {
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.viewOf = await this.viewOf.removeObsAnnoCategory(
+      col,
+      category,
+      unassignedCategory
+    );
+    newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema;
+    return newAnnoMatrix;
+  }
+
+  dropObsColumn(col) {
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.viewOf = this.viewOf.dropObsColumn(col);
+    newAnnoMatrix._cache.obs = this._cache.obs.dropCol(col);
+    newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema;
+    return newAnnoMatrix;
+  }
+
+  addObsColumn(colSchema, Ctor, value) {
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.viewOf = this.viewOf.addObsColumn(colSchema, Ctor, value);
+    newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema;
+    return newAnnoMatrix;
+  }
+
+  renameObsColumn(oldCol, newCol) {
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.viewOf = this.viewOf.renameObsColumn(oldCol, newCol);
+    newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema;
+    return newAnnoMatrix;
+  }
+
+  async setObsColumnValues(col, rowLabels, value) {
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.viewOf = await this.viewOf.setObsColumnValues(
+      col,
+      rowLabels,
+      value
+    );
+    newAnnoMatrix._cache.obs = this._cache.obs.dropCol(col);
+    newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema;
+    return newAnnoMatrix;
+  }
+
+  async resetObsColumnValues(col, oldValue, newValue) {
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.viewOf = await this.viewOf.resetObsColumnValues(
+      col,
+      oldValue,
+      newValue
+    );
+    newAnnoMatrix._cache.obs = this._cache.obs.dropCol(col);
+    newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema;
+    return newAnnoMatrix;
+  }
+
+  addEmbedding(colSchema) {
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.viewOf = this.viewOf.addEmbedding(colSchema);
+    newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema;
+    return newAnnoMatrix;
+  }
+}
+
+class AnnoMatrixMapView extends AnnoMatrixView {
+  /*
+	A view which knows how to transform its data.
+	*/
+  constructor(viewOf, mapFn) {
+    super(viewOf);
+    this.mapFn = mapFn;
+  }
+
+  async _doLoad(field, query) {
+    const df = await this.viewOf._fetch(field, query);
+    const dfMapped = df.mapColumns((colData, colIdx) => {
+      const colLabel = df.colIndex.getLabel(colIdx);
+      const colSchema = _getColumnSchema(this.schema, field, colLabel);
+      return this.mapFn(field, colLabel, colSchema, colData, df);
+    });
+    const whereCacheUpdate = _whereCacheCreate(
+      field,
+      query,
+      dfMapped.colIndex.labels()
+    );
+    return [whereCacheUpdate, dfMapped];
+  }
+}
+
+export class AnnoMatrixClipView extends AnnoMatrixMapView {
+  /*
+	A view which is a clipped transformation of its parent
+	*/
+  constructor(viewOf, qmin, qmax) {
+    super(viewOf, (field, colLabel, colSchema, colData, df) =>
+      _clipAnnoMatrix(field, colLabel, colSchema, colData, df, qmin, qmax)
+    );
+    this.isClipped = true;
+    this.clipRange = [qmin, qmax];
+    Object.seal(this);
+  }
+}
+
+export class AnnoMatrixRowSubsetView extends AnnoMatrixView {
+  /*
+	A view which is a subset of total rows.
+	*/
+  constructor(viewOf, rowIndex) {
+    super(viewOf, rowIndex);
+    Object.seal(this);
+  }
+
+  async _doLoad(field, query) {
+    const df = await this.viewOf._fetch(field, query);
+
+    // don't try to row-subset the var dimension.
+    if (field === "var") {
+      return [null, df];
+    }
+
+    const dfSubset = df.subset(null, null, this.rowIndex);
+    const whereCacheUpdate = _whereCacheCreate(
+      field,
+      query,
+      dfSubset.colIndex.labels()
+    );
+    return [whereCacheUpdate, dfSubset];
+  }
+}
+
+/*
+Utility functions below
+*/
+
+function _clipAnnoMatrix(field, colLabel, colSchema, colData, df, qmin, qmax) {
+  /* only clip obs and var scalar columns */
+  if (field !== "obs" && field !== "X") return colData;
+  if (!_isContinuousType(colSchema)) return colData;
+  if (qmin < 0) qmin = 0;
+  if (qmax > 1) qmax = 1;
+  if (qmin === 0 && qmax === 1) return colData;
+
+  const quantiles = df.col(colLabel).summarize().percentiles;
+  const lower = quantiles[100 * qmin];
+  const upper = quantiles[100 * qmax];
+  const clippedData = clip(colData.slice(), lower, upper, Number.NaN);
+  return clippedData;
+}
+
+/* eslint-enable max-classes-per-file -- enable*/

client/src/annoMatrix/whereCache.js

@@ -0,0 +1,92 @@
+/*
+Private support functions.
+
+Support for a "where" query, eg,
+
+	{ where: { field: "var", column: "gene", value: "FOXP2" }}
+
+These evaluate to a given column label.
+
+The "where cache" is a map that saves evaluated queries and points 
+to the column label they resolve to.
+
+Data structure, using X as the example field being queried, and var as
+the index.
+
+{ 
+	X: {
+		var: Map(
+			column_label_in_var => Map(value_in_var_column => [column_label_in_X, ...])
+		)
+	}
+}
+*/
+import { _getColumnDimensionNames } from "./schema";
+
+export function _whereCacheGet(whereCache, schema, field, query) {
+  /* 
+	query will either be an where query (object) or a column name (string).
+
+	Return array of column labels or undefined.
+	*/
+
+  if (typeof query === "object") {
+    const { field: queryField, column: queryColumn, value: queryValue } = query;
+
+    const columnMap = whereCache?.[field]?.[queryField];
+    if (columnMap === undefined) return [undefined];
+
+    const valueMap = columnMap.get(queryColumn);
+    if (valueMap === undefined) return [undefined];
+
+    const columnLabels = valueMap.get(queryValue);
+    return columnLabels === undefined ? [undefined] : columnLabels;
+  }
+
+  const colDims = _getColumnDimensionNames(schema, field, query);
+  return colDims === undefined ? [undefined] : colDims;
+}
+
+export function _whereCacheCreate(field, query, columnLabels) {
+  /*
+	Create a new whereCache
+	*/
+  if (typeof query !== "object") return null;
+
+  const { field: queryField, column: queryColumn, value: queryValue } = query;
+  const whereCache = {
+    [field]: {
+      [queryField]: new Map([
+        [queryColumn, new Map([[queryValue, columnLabels]])],
+      ]),
+    },
+  };
+  return whereCache;
+}
+
+function __whereCacheMerge(dst, src) {
+  /*
+	merge src into dst (modifies dst)
+	*/
+  if (!dst) dst = {};
+  if (!src || typeof src !== "object") return dst;
+  Object.entries(src).forEach(([field, query]) => {
+    if (!Object.prototype.hasOwnProperty.call(dst, field)) dst[field] = {};
+    Object.entries(query).forEach(([queryField, columnMap]) => {
+      if (!Object.prototype.hasOwnProperty.call(dst[field], queryField))
+        dst[field][queryField] = new Map();
+      columnMap.forEach((valueMap, queryColumn) => {
+        if (!dst[field][queryField].has(queryColumn))
+          dst[field][queryField].set(queryColumn, new Map());
+        valueMap.forEach((columnLabels, queryValue) => {
+          dst[field][queryField].get(queryColumn).set(queryValue, columnLabels);
+        });
+      });
+    });
+  });
+  return dst;
+}
+
+export function _whereCacheMerge(...caches) {
+  return caches.reduce((dst, src) => __whereCacheMerge(dst, src), {});
+}

client/src/components/app.js

@@ -0,0 +1,92 @@
+// jshint esversion: 6
+import React from "react";
+import Helmet from "react-helmet";
+import { connect } from "react-redux";
+
+import Container from "./framework/container";
+import Layout from "./framework/layout";
+import LeftSideBar from "./leftSidebar";
+import RightSideBar from "./rightSidebar";
+import Legend from "./continuousLegend";
+import Graph from "./graph/graph";
+import MenuBar from "./menubar";
+import Autosave from "./autosave";
+import Embedding from "./embedding";
+import TermsOfServicePrompt from "./termsPrompt";
+
+import actions from "../actions";
+
+@connect((state) => ({
+  loading: state.controls.loading,
+  error: state.controls.error,
+  graphRenderCounter: state.controls.graphRenderCounter,
+}))
+class App extends React.Component {
+  componentDidMount() {
+    const { dispatch } = this.props;
+
+    /* listen for url changes, fire one when we start the app up */
+    window.addEventListener("popstate", this._onURLChanged);
+    this._onURLChanged();
+
+    dispatch(actions.doInitialDataLoad(window.location.search));
+    this.forceUpdate();
+  }
+
+  _onURLChanged() {
+    const { dispatch } = this.props;
+
+    dispatch({ type: "url changed", url: document.location.href });
+  }
+
+  render() {
+    const { loading, error, graphRenderCounter } = this.props;
+    return (
+      <Container>
+        <Helmet title="cellxgene" />
+        {loading ? (
+          <div
+            style={{
+              position: "fixed",
+              fontWeight: 500,
+              top: window.innerHeight / 2,
+              left: window.innerWidth / 2 - 50,
+            }}
+          >
+            loading cellxgene
+          </div>
+        ) : null}
+        {error ? (
+          <div
+            style={{
+              position: "fixed",
+              fontWeight: 500,
+              top: window.innerHeight / 2,
+              left: window.innerWidth / 2 - 50,
+            }}
+          >
+            error loading cellxgene
+          </div>
+        ) : null}
+        {loading || error ? null : (
+          <Layout>
+            <LeftSideBar />
+            {(viewportRef) => (
+              <>
+                <MenuBar />
+                <Embedding />
+                <Autosave />
+                <TermsOfServicePrompt />
+                <Legend viewportRef={viewportRef} />
+                <Graph key={graphRenderCounter} viewportRef={viewportRef} />
+              </>
+            )}
+            <RightSideBar />
+          </Layout>
+        )}
+      </Container>
+    );
+  }
+}
+
+export default App;

client/src/components/autosave/filenameDialog.js

@@ -0,0 +1,172 @@
+import React from "react";
+import { connect } from "react-redux";
+
+import {
+  Button,
+  Tooltip,
+  InputGroup,
+  Dialog,
+  Classes,
+  Colors,
+} from "@blueprintjs/core";
+
+@connect((state) => ({
+  idhash: state.config?.parameters?.["annotations-user-data-idhash"] ?? null,
+  annotations: state.annotations,
+  auth: state.config?.authentication,
+  writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
+}))
+class FilenameDialog extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = {
+      filenameText: "",
+    };
+  }
+
+  dismissFilenameDialog = () => {};
+
+  handleCreateFilename = () => {
+    const { dispatch } = this.props;
+    const { filenameText } = this.state;
+
+    dispatch({
+      type: "set annotations collection name",
+      data: filenameText,
+    });
+  };
+
+  filenameError = () => {
+    const legalNames = /^\w+$/;
+    const { filenameText } = this.state;
+    let err = false;
+
+    if (filenameText === "") {
+      err = "empty_string";
+    } else if (!legalNames.test(filenameText)) {
+      /*
+      IMPORTANT: this test must ultimately match the test applied by the
+      backend, which is designed to ensure a safe file name can be created
+      from the data collection name.  If you change this, you will also need
+      to change the validation code in the backend, or it will have no effect.
+      */
+      err = "characters";
+    }
+
+    return err;
+  };
+
+  filenameErrorMessage = () => {
+    const err = this.filenameError();
+    let markup = null;
+
+    if (err === "empty_string") {
+      markup = (
+        <span
+          style={{
+            fontStyle: "italic",
+            fontSize: 12,
+            marginTop: 5,
+            color: Colors.ORANGE3,
+          }}
+        >
+          Name cannot be blank
+        </span>
+      );
+    } else if (err === "characters") {
+      markup = (
+        <span
+          style={{
+            fontStyle: "italic",
+            fontSize: 12,
+            marginTop: 5,
+            color: Colors.ORANGE3,
+          }}
+        >
+          Only alphanumeric and underscore allowed
+        </span>
+      );
+    }
+    return markup;
+  };
+
+  render() {
+    const { writableCategoriesEnabled, annotations, idhash, auth } = this.props;
+    const { filenameText } = this.state;
+
+    return writableCategoriesEnabled &&
+      !annotations.dataCollectionNameIsReadOnly &&
+      !annotations.dataCollectionName &&
+      auth.is_authenticated ? (
+      <Dialog
+        icon="tag"
+        title="Annotations Collection"
+        isOpen={!annotations.dataCollectionName}
+        onClose={this.dismissFilenameDialog}
+      >
+        <form
+          onSubmit={(e) => {
+            e.preventDefault();
+            this.handleCreateFilename();
+          }}
+        >
+          <div className={Classes.DIALOG_BODY} data-testid="annotation-dialog">
+            <div style={{ marginBottom: 20 }}>
+              <p>Name your annotations collection:</p>
+              <InputGroup
+                autoFocus
+                value={filenameText}
+                intent={this.filenameError(filenameText) ? "warning" : "none"}
+                onChange={(e) =>
+                  this.setState({ filenameText: e.target.value })
+                }
+                leftIcon="tag"
+                data-testid="new-annotation-name"
+              />
+              <p
+                style={{
+                  marginTop: 7,
+                  visibility: this.filenameError(filenameText)
+                    ? "visible"
+                    : "hidden",
+                  color: Colors.ORANGE3,
+                }}
+              >
+                {this.filenameErrorMessage(filenameText)}
+              </p>
+            </div>
+            <div>
+              <p>
+                Your annotations are stored in this file:
+                <code className="bp3-code">
+                  {filenameText}-{idhash}.csv
+                </code>
+              </p>
+              <p style={{ fontStyle: "italic" }}>
+                (We added a unique ID to your filename)
+              </p>
+            </div>
+          </div>
+          <div className={Classes.DIALOG_FOOTER}>
+            <div className={Classes.DIALOG_FOOTER_ACTIONS}>
+              <Tooltip content="Cancel naming collection">
+                <Button onClick={this.dismissFilenameDialog}>Cancel</Button>
+              </Tooltip>
+              <Button
+                disabled={!filenameText || this.filenameError(filenameText)}
+                onClick={this.handleCreateFilename}
+                intent="primary"
+                type="submit"
+                data-testid="submit-annotation"
+              >
+                Create annotations collection
+              </Button>
+            </div>
+          </div>
+        </form>
+      </Dialog>
+    ) : null;
+  }
+}
+
+export default FilenameDialog;

client/src/components/autosave/index.js

@@ -0,0 +1,96 @@
+import React from "react";
+import { connect } from "react-redux";
+import actions from "../../actions";
+import FilenameDialog from "./filenameDialog";
+
+@connect((state) => ({
+  annotations: state.annotations,
+  saveInProgress: state.autosave?.saveInProgress ?? false,
+  error: state.autosave?.error,
+  writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
+  annoMatrix: state.annoMatrix,
+  lastSavedAnnoMatrix: state.autosave?.lastSavedAnnoMatrix,
+}))
+class Autosave extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = {
+      timer: null,
+    };
+  }
+
+  componentDidMount() {
+    const { writableCategoriesEnabled } = this.props;
+
+    let { timer } = this.state;
+    if (timer) clearInterval(timer);
+    if (writableCategoriesEnabled) {
+      timer = setInterval(this.tick, 2500);
+    } else {
+      timer = null;
+    }
+    this.setState({ timer });
+  }
+
+  componentWillUnmount() {
+    const { timer } = this.state;
+    if (timer) this.clearInterval(timer);
+  }
+
+  tick = () => {
+    const { dispatch, saveInProgress } = this.props;
+    if (this.needToSave() && !saveInProgress) {
+      dispatch(actions.saveObsAnnotationsAction());
+    }
+  };
+
+  needToSave = () => {
+    /* return true if we need to save, false if we don't */
+    const { annoMatrix, lastSavedAnnoMatrix } = this.props;
+    return actions.needToSaveObsAnnotations(annoMatrix, lastSavedAnnoMatrix);
+  };
+
+  statusMessage() {
+    const { error } = this.props;
+    if (error) {
+      return `Autosave error: ${error}`;
+    }
+    return this.needToSave() ? "Unsaved" : "All saved";
+  }
+
+  render() {
+    const {
+      writableCategoriesEnabled,
+      saveInProgress,
+      lastSavedAnnoMatrix,
+    } = this.props;
+    const initialDataLoadComplete = lastSavedAnnoMatrix;
+
+    if (!writableCategoriesEnabled) return null;
+
+    return (
+      <div
+        id="autosave"
+        data-testclass={
+          !initialDataLoadComplete
+            ? "autosave-init"
+            : this.needToSave() || saveInProgress
+            ? "autosave-incomplete"
+            : "autosave-complete"
+        }
+        style={{
+          position: "absolute",
+          display: "inherit",
+          right: 8,
+          bottom: 8,
+          zIndex: 1,
+        }}
+      >
+        {this.statusMessage()}
+        <FilenameDialog />
+      </div>
+    );
+  }
+}
+
+export default Autosave;

client/src/components/brushableHistogram/index.js

@@ -0,0 +1,817 @@
+/*
+https://bl.ocks.org/mbostock/4341954
+https://bl.ocks.org/mbostock/34f08d5e11952a80609169b7917d4172
+https://bl.ocks.org/SpaceActuary/2f004899ea1b2bd78d6f1dbb2febf771
+https://bl.ocks.org/mbostock/3019563
+*/
+import React, { useEffect, useRef, useState, useCallback } from "react";
+import { Button, ButtonGroup, Tooltip } from "@blueprintjs/core";
+import { connect } from "react-redux";
+import * as d3 from "d3";
+import { interpolateCool } from "d3-scale-chromatic";
+import Async from "react-async";
+import memoize from "memoize-one";
+import * as globals from "../../globals";
+import actions from "../../actions";
+import { histogramContinuous } from "../../util/dataframe/histogram";
+import { makeContinuousDimensionName } from "../../util/nameCreators";
+import significantDigits from "../../util/significantDigits";
+
+function clamp(val, rng) {
+  return Math.max(Math.min(val, rng[1]), rng[0]);
+}
+
+function maybeScientific(x) {
+  let format = ",";
+  const _ticks = x.ticks(4);
+
+  if (x.domain().some((n) => Math.abs(n) >= 10000)) {
+    /* 
+      heuristic: if the last tick d3 wants to render has one significant
+      digit ie., 2000, render 2e+3, but if it's anything else ie., 42000000 render
+      4.20e+n
+    */
+    format = significantDigits(_ticks[_ticks.length - 1]) === 1 ? ".0e" : ".2e";
+  }
+
+  return format;
+}
+
+const StillLoading = ({ zebra, displayName }) => {
+  /*
+  Render a loading indicator for the field.
+  */
+  return (
+    <div
+      style={{
+        padding: globals.leftSidebarSectionPadding,
+        backgroundColor: zebra ? globals.lightestGrey : "white",
+      }}
+    >
+      <div
+        style={{
+          display: "flex",
+          justifyContent: "space-between",
+          justifyItems: "center",
+          alignItems: "center",
+        }}
+      >
+        <div style={{ minWidth: 30 }} />
+        <div style={{ display: "flex", alignSelf: "center" }}>
+          <span style={{ fontStyle: "italic" }}>{displayName}</span>
+        </div>
+        <div
+          style={{
+            display: "flex",
+            justifyContent: "flex-end",
+          }}
+        >
+          <Button minimal loading intent="primary" />
+        </div>
+      </div>
+    </div>
+  );
+};
+
+const ErrorLoading = ({ displayName, error, zebra }) => {
+  console.log(error); // log to console as this is unexpected
+  return (
+    <div
+      style={{
+        padding: globals.leftSidebarSectionPadding,
+        backgroundColor: zebra ? globals.lightestGrey : "white",
+      }}
+    >
+      <span>{`Failure loading ${displayName}`}</span>
+    </div>
+  );
+};
+
+const HistogramFooter = React.memo(
+  ({
+    displayName,
+    hideRanges,
+    rangeMin,
+    rangeMax,
+    rangeColorMin,
+    rangeColorMax,
+    logFoldChange,
+    pvalAdj,
+  }) => {
+    /*
+  Footer of each histogram.  Will render range, title, and optionally 
+  differential expression info.
+
+  Required props:
+    * displayName - the displayName, aka "n_genes", "FOXP2", etc.
+    * hideRanges - true/false, enables/disable rendering of ranges
+    * range - length two array, [min, max], containing the range values to display
+    * rangeColor - length two array, [mincolor, maxcolor], each a CSS color
+    * logFoldChange - lfc to display, optional.
+    * pValue - pValue to display, optional.
+  */
+    return (
+      <div>
+        <div
+          style={{
+            display: "flex",
+            justifyContent: hideRanges ? "center" : "space-between",
+          }}
+        >
+          <span
+            style={{
+              color: rangeColorMin,
+              display: hideRanges ? "none" : "block",
+            }}
+          >
+            min {rangeMin.toPrecision(4)}
+          </span>
+          <span
+            data-testclass="brushable-histogram-field-name"
+            style={{ fontStyle: "italic" }}
+          >
+            {displayName}
+          </span>
+          <div style={{ display: hideRanges ? "block" : "none" }}>
+            : {rangeMin}
+          </div>
+          <span
+            style={{
+              color: rangeColorMax,
+              display: hideRanges ? "none" : "block",
+            }}
+          >
+            max {rangeMax.toPrecision(4)}
+          </span>
+        </div>
+
+        {logFoldChange && pvalAdj ? (
+          <div
+            style={{
+              display: "flex",
+              justifyContent: "center",
+              alignItems: "baseline",
+            }}
+          >
+            <span>
+              <strong>log fold change:</strong>
+              {` ${logFoldChange.toPrecision(4)}`}
+            </span>
+            <span
+              style={{
+                marginLeft: 7,
+                padding: 2,
+              }}
+            >
+              <strong>p-value (adj):</strong>
+              {pvalAdj < 0.0001 ? " < 0.0001" : ` ${pvalAdj.toFixed(4)}`}
+            </span>
+          </div>
+        ) : null}
+      </div>
+    );
+  }
+);
+
+const HistogramHeader = React.memo(
+  ({
+    fieldId,
+    isColorBy,
+    onColorByClick,
+    onRemoveClick,
+    isScatterPlotX,
+    isScatterPlotY,
+    onScatterPlotXClick,
+    onScatterPlotYClick,
+    isObs,
+  }) => {
+    /*
+      Render the toolbar for the histogram.  Props:
+        * fieldId - field identifier, used for various IDs
+        * isColorBy - true/false, is this the current color-by
+        * onColorByClick - color-by click handler
+        * onRemoveClick - optional handler for remove.  Button will not render if not defined.
+        * isScatterPlotX - optional, true/false if currently the X scatterplot field
+        * isScatterPlotY - optional, true/false if currently the Y scatterplot field
+        * onScatterPlotXClick - optional, handler for scatterPlot X button.
+        * onScatterPlotYClick - optional, handler for scatterPlot X button.
+
+      Scatterplot controls will not render if either handler unspecified.
+    */
+
+    const memoizedColorByCallback = useCallback(
+      () => onColorByClick(fieldId, isObs),
+      [fieldId, isObs]
+    );
+
+    return (
+      <div
+        style={{
+          display: "flex",
+          justifyContent: "flex-end",
+          paddingBottom: "8px",
+        }}
+      >
+        {onScatterPlotXClick && onScatterPlotYClick ? (
+          <span>
+            <span
+              style={{ marginRight: 7 }}
+              className="bp3-icon-standard bp3-icon-scatter-plot"
+            />
+            <ButtonGroup style={{ marginRight: 7 }}>
+              <Button
+                data-testid={`plot-x-${fieldId}`}
+                onClick={onScatterPlotXClick}
+                active={isScatterPlotX}
+                intent={isScatterPlotX ? "primary" : "none"}
+              >
+                plot x
+              </Button>
+              <Button
+                data-testid={`plot-y-${fieldId}`}
+                onClick={onScatterPlotYClick}
+                active={isScatterPlotY}
+                intent={isScatterPlotY ? "primary" : "none"}
+              >
+                plot y
+              </Button>
+            </ButtonGroup>
+          </span>
+        ) : null}
+        {onRemoveClick ? (
+          <Button
+            minimal
+            onClick={onRemoveClick}
+            style={{
+              color: globals.blue,
+              cursor: "pointer",
+              marginLeft: 7,
+            }}
+          >
+            remove
+          </Button>
+        ) : null}
+        <Tooltip
+          content="Use as color scale"
+          position="bottom"
+          hoverOpenDelay={globals.tooltipHoverOpenDelay}
+        >
+          <Button
+            onClick={memoizedColorByCallback}
+            active={isColorBy}
+            intent={isColorBy ? "primary" : "none"}
+            data-testclass="colorby"
+            data-testid={`colorby-${fieldId}`}
+            icon="tint"
+          />
+        </Tooltip>
+      </div>
+    );
+  }
+);
+
+const Histogram = ({
+  field,
+  fieldForId,
+  display,
+  histogram,
+  width,
+  height,
+  onBrush,
+  onBrushEnd,
+  margin,
+  isColorBy,
+  selectionRange,
+}) => {
+  const svgRef = useRef(null);
+  const [brush, setBrush] = useState(null);
+
+  useEffect(() => {
+    /*
+    Create the d3 histogram
+    */
+    const { marginLeft, marginRight, marginBottom, marginTop } = margin;
+    const { x, y, bins, binStart, binEnd, binWidth } = histogram;
+    const svg = d3.select(svgRef.current);
+
+    /* Remove everything */
+    svg.selectAll("*").remove();
+
+    /* Set margins within the SVG */
+    const container = svg
+      .attr("width", width + marginLeft + marginRight)
+      .attr("height", height + marginTop + marginBottom)
+      .append("g")
+      .attr("class", "histogram-container")
+      .attr("transform", `translate(${marginLeft},${marginTop})`);
+
+    const colorScale = d3
+      .scaleSequential(interpolateCool)
+      .domain([0, bins.length]);
+
+    const histogramScale = d3
+      .scaleLinear()
+      .domain(x.domain())
+      .range([
+        colorScale.domain()[1],
+        colorScale.domain()[0],
+      ]); /* we flip this to make colors dark if high in the color scale */
+
+    if (binWidth > 0) {
+      /* BINS */
+      container
+        .insert("g", "*")
+        .selectAll("rect")
+        .data(bins)
+        .enter()
+        .append("rect")
+        .attr("x", (d, i) => x(binStart(i)) + 1)
+        .attr("y", (d) => y(d))
+        .attr("width", (d, i) => x(binEnd(i)) - x(binStart(i)) - 1)
+        .attr("height", (d) => y(0) - y(d))
+        .style(
+          "fill",
+          isColorBy ? (d, i) => colorScale(histogramScale(binStart(i))) : "#bbb"
+        );
+    }
+
+    // BRUSH
+    // Note the brushable area is bounded by the data on three sides, but goes down to cover the x-axis
+    const brushX = d3
+      .brushX()
+      .extent([
+        [x.range()[0], y.range()[1]],
+        [x.range()[1], marginTop + height + marginBottom],
+      ])
+      /*
+      emit start so that the Undoable history can save an undo point
+      upon drag start, and ignore the subsequent intermediate drag events.
+      */
+      .on("start", onBrush(field, x.invert, "start"))
+      .on("brush", onBrush(field, x.invert, "brush"))
+      .on("end", onBrushEnd(field, x.invert));
+
+    const brushXselection = container
+      .insert("g")
+      .attr("class", "brush")
+      .attr("data-testid", `${svgRef.current.dataset.testid}-brushable-area`)
+      .call(brushX);
+
+    /* X AXIS */
+    container
+      .insert("g")
+      .attr("class", "axis axis--x")
+      .attr("transform", `translate(0,${marginTop + height})`)
+      .call(
+        d3
+          .axisBottom(x)
+          .ticks(4)
+          .tickFormat(d3.format(maybeScientific(x)))
+      );
+
+    /* Y AXIS */
+    container
+      .insert("g")
+      .attr("class", "axis axis--y")
+      .attr("transform", `translate(${marginLeft + width},0)`)
+      .call(
+        d3
+          .axisRight(y)
+          .ticks(3)
+          .tickFormat(
+            d3.format(
+              y.domain().some((n) => Math.abs(n) >= 10000) ? ".0e" : ","
+            )
+          )
+      );
+
+    /* axis style */
+    svg.selectAll(".axis text").style("fill", "rgb(80,80,80)");
+    svg.selectAll(".axis path").style("stroke", "rgb(230,230,230)");
+    svg.selectAll(".axis line").style("stroke", "rgb(230,230,230)");
+
+    setBrush({ brushX, brushXselection });
+  }, [histogram, isColorBy]);
+
+  useEffect(() => {
+    /*
+    paint/update selection brush
+    */
+    if (!brush) return;
+    const { brushX, brushXselection } = brush;
+    const selection = d3.brushSelection(brushXselection.node());
+    if (!selectionRange && selection) {
+      /* no active selection - clear brush */
+      brushXselection.call(brushX.move, null);
+    } else if (selectionRange) {
+      const { x, domain } = histogram;
+      const [min, max] = domain;
+      const x0 = x(clamp(selectionRange[0], [min, max]));
+      const x1 = x(clamp(selectionRange[1], [min, max]));
+      if (!selection) {
+        /* there is an active selection, but no brush - set the brush */
+        brushXselection.call(brushX.move, [x0, x1]);
+      } else {
+        /* there is an active selection and a brush - make sure they match */
+        const moveDeltaThreshold = 1;
+        const dX0 = Math.abs(x0 - selection[0]);
+        const dX1 = Math.abs(x1 - selection[1]);
+        /*
+        only update the brush if it is grossly incorrect,
+        as defined by the moveDeltaThreshold
+        */
+        if (dX0 > moveDeltaThreshold || dX1 > moveDeltaThreshold) {
+          brushXselection.call(brushX.move, [x0, x1]);
+        }
+      }
+    }
+  }, [brush, selectionRange]);
+
+  return (
+    <svg
+      style={{ display }}
+      width={width}
+      height={height}
+      id={`histogram_${fieldForId}_svg`}
+      data-testclass="histogram-plot"
+      data-testid={`histogram-${field}-plot`}
+      ref={svgRef}
+    />
+  );
+};
+
+@connect((state, ownProps) => {
+  const { isObs, isUserDefined, isDiffExp, field } = ownProps;
+  const myName = makeContinuousDimensionName(
+    { isObs, isUserDefined, isDiffExp },
+    field
+  );
+  return {
+    annoMatrix: state.annoMatrix,
+    isScatterplotXXaccessor: state.controls.scatterplotXXaccessor === field,
+    isScatterplotYYaccessor: state.controls.scatterplotYYaccessor === field,
+    continuousSelectionRange: state.continuousSelection[myName],
+    isColorAccessor: state.colors.colorAccessor === field,
+  };
+})
+class HistogramBrush extends React.PureComponent {
+  /* memoized closure to prevent HistogramHeader unecessary repaint */
+  handleColorAction = memoize((dispatch) => (field, isObs) => {
+    if (isObs) {
+      dispatch({
+        type: "color by continuous metadata",
+        colorAccessor: field,
+      });
+    } else {
+      dispatch(actions.requestSingleGeneExpressionCountsForColoringPOST(field));
+    }
+  });
+
+  constructor(props) {
+    super(props);
+
+    const marginLeft = 10; // Space for 0 tick label on X axis
+    const marginRight = 54; // space for Y axis & labels
+    const marginBottom = 25; // space for X axis & labels
+    const marginTop = 3;
+    this.margin = {
+      marginLeft,
+      marginRight,
+      marginBottom,
+      marginTop,
+    };
+    this.width = 340 - marginLeft - marginRight;
+    this.height = 135 - marginTop - marginBottom;
+  }
+
+  onBrush = (selection, x, eventType) => {
+    const type = `continuous metadata histogram ${eventType}`;
+    return () => {
+      const { dispatch, field, isObs, isUserDefined, isDiffExp } = this.props;
+
+      // ignore programmatically generated events
+      if (!d3.event.sourceEvent) return;
+      // ignore cascading events, which are programmatically generated
+      if (d3.event.sourceEvent.sourceEvent) return;
+
+      const query = this.createQuery();
+      const range = d3.event.selection
+        ? [x(d3.event.selection[0]), x(d3.event.selection[1])]
+        : null;
+      const otherProps = {
+        selection: field,
+        continuousNamespace: {
+          isObs,
+          isUserDefined,
+          isDiffExp,
+        },
+      };
+      dispatch(
+        actions.selectContinuousMetadataAction(type, query, range, otherProps)
+      );
+    };
+  };
+
+  onBrushEnd = (selection, x) => {
+    return () => {
+      const { dispatch, field, isObs, isUserDefined, isDiffExp } = this.props;
+      const minAllowedBrushSize = 10;
+      const smallAmountToAvoidInfiniteLoop = 0.1;
+
+      // ignore programmatically generated events
+      if (!d3.event.sourceEvent) return;
+      // ignore cascading events, which are programmatically generated
+      if (d3.event.sourceEvent.sourceEvent) return;
+
+      let type;
+      let range = null;
+      if (d3.event.selection) {
+        type = "continuous metadata histogram end";
+        if (
+          d3.event.selection[1] - d3.event.selection[0] >
+          minAllowedBrushSize
+        ) {
+          range = [x(d3.event.selection[0]), x(d3.event.selection[1])];
+        } else {
+          /* the user selected range is too small and will be hidden #587, so take control of it procedurally */
+          /* https://stackoverflow.com/questions/12354729/d3-js-limit-size-of-brush */
+
+          const procedurallyResizedBrushWidth =
+            d3.event.selection[0] +
+            minAllowedBrushSize +
+            smallAmountToAvoidInfiniteLoop; //
+
+          range = [x(d3.event.selection[0]), x(procedurallyResizedBrushWidth)];
+        }
+      } else {
+        type = "continuous metadata histogram cancel";
+      }
+
+      const query = this.createQuery();
+      const otherProps = {
+        selection: field,
+        continuousNamespace: {
+          isObs,
+          isUserDefined,
+          isDiffExp,
+        },
+      };
+      dispatch(
+        actions.selectContinuousMetadataAction(type, query, range, otherProps)
+      );
+    };
+  };
+
+  handleSetGeneAsScatterplotX = () => {
+    const { dispatch, field } = this.props;
+    dispatch({
+      type: "set scatterplot x",
+      data: field,
+    });
+  };
+
+  handleSetGeneAsScatterplotY = () => {
+    const { dispatch, field } = this.props;
+    dispatch({
+      type: "set scatterplot y",
+      data: field,
+    });
+  };
+
+  removeHistogram = () => {
+    const {
+      dispatch,
+      field,
+      isColorAccessor,
+      isScatterplotXXaccessor,
+      isScatterplotYYaccessor,
+    } = this.props;
+    dispatch({
+      type: "clear user defined gene",
+      data: field,
+    });
+    if (isColorAccessor) {
+      dispatch({
+        type: "reset colorscale",
+      });
+    }
+    if (isScatterplotXXaccessor) {
+      dispatch({
+        type: "set scatterplot x",
+        data: null,
+      });
+    }
+    if (isScatterplotYYaccessor) {
+      dispatch({
+        type: "set scatterplot y",
+        data: null,
+      });
+    }
+  };
+
+  fetchAsyncProps = async () => {
+    const { annoMatrix } = this.props;
+    const { isClipped } = annoMatrix;
+
+    const query = this.createQuery();
+    const df = await annoMatrix.fetch(...query);
+    const column = df.icol(0);
+
+    // if we are clipped, fetch both our value and our unclipped value,
+    // as we need the absolute min/max range, not just the clipped min/max.
+    const summary = column.summarize();
+    const range = [summary.min, summary.max];
+
+    let unclippedRange = [...range];
+    if (isClipped) {
+      const parent = await annoMatrix.viewOf.fetch(...query);
+      const { min, max } = parent.icol(0).summarize();
+      unclippedRange = [min, max];
+    }
+
+    const unclippedRangeColor = [
+      !annoMatrix.isClipped || annoMatrix.clipRange[0] === 0
+        ? "#bbb"
+        : globals.blue,
+      !annoMatrix.isClipped || annoMatrix.clipRange[1] === 1
+        ? "#bbb"
+        : globals.blue,
+    ];
+
+    const histogram = this.calcHistogramCache(
+      column,
+      this.margin,
+      this.width,
+      this.height
+    );
+
+    const isSingleValue = summary.min === summary.max;
+    const nonFiniteExtent =
+      summary.min === undefined ||
+      summary.max === undefined ||
+      Number.isNaN(summary.min) ||
+      Number.isNaN(summary.max);
+
+    const OK2Render = !summary.categorical && !nonFiniteExtent;
+
+    return {
+      histogram,
+      range,
+      unclippedRange,
+      unclippedRangeColor,
+      isSingleValue,
+      OK2Render,
+    };
+  };
+
+  // eslint-disable-next-line class-methods-use-this -- instance method allows for memoization per annotation
+  calcHistogramCache(col, margin, width, height) {
+    /*
+     recalculate expensive stuff, notably bins, summaries, etc.
+    */
+    const histogramCache = {};
+    const summary = col.summarize();
+    const { min: domainMin, max: domainMax } = summary;
+    const numBins = 40;
+    const { marginTop, marginLeft } = margin;
+
+    histogramCache.domain = [domainMin, domainMax];
+
+    histogramCache.x = d3
+      .scaleLinear()
+      .domain([domainMin, domainMax])
+      .range([marginLeft, marginLeft + width]);
+
+    histogramCache.bins = histogramContinuous(col, numBins, [
+      domainMin,
+      domainMax,
+    ]);
+    histogramCache.binWidth = (domainMax - domainMin) / numBins;
+
+    histogramCache.binStart = (i) => domainMin + i * histogramCache.binWidth;
+    histogramCache.binEnd = (i) =>
+      domainMin + (i + 1) * histogramCache.binWidth;
+
+    const yMax = histogramCache.bins.reduce((l, r) => (l > r ? l : r));
+
+    histogramCache.y = d3
+      .scaleLinear()
+      .domain([0, yMax])
+      .range([marginTop + height, marginTop]);
+
+    return histogramCache;
+  }
+
+  createQuery() {
+    const { isObs, field, annoMatrix } = this.props;
+    const { schema } = annoMatrix;
+    if (isObs) {
+      return ["obs", field];
+    }
+    const varIndex = schema?.annotations?.var?.index;
+    if (!varIndex) return null;
+    return [
+      "X",
+      {
+        field: "var",
+        column: varIndex,
+        value: field,
+      },
+    ];
+  }
+
+  render() {
+    const {
+      dispatch,
+      annoMatrix,
+      field,
+      isColorAccessor,
+      isUserDefined,
+      isDiffExp,
+      logFoldChange,
+      pvalAdj,
+      isScatterplotXXaccessor,
+      isScatterplotYYaccessor,
+      zebra,
+      continuousSelectionRange,
+      isObs,
+    } = this.props;
+    const fieldForId = field.replace(/\s/g, "_");
+    const showScatterPlot = isDiffExp || isUserDefined;
+
+    return (
+      <Async watch={annoMatrix} promiseFn={this.fetchAsyncProps}>
+        <Async.Pending initial>
+          <StillLoading displayName={field} zebra={zebra} />
+        </Async.Pending>
+        <Async.Rejected>
+          {(error) => (
+            <ErrorLoading zebra={zebra} error={error} displayName={field} />
+          )}
+        </Async.Rejected>
+        <Async.Fulfilled>
+          {(asyncProps) =>
+            asyncProps.OK2Render ? (
+              <div
+                id={`histogram_${fieldForId}`}
+                data-testid={`histogram-${field}`}
+                data-testclass={
+                  isDiffExp
+                    ? "histogram-diffexp"
+                    : isUserDefined
+                    ? "histogram-user-gene"
+                    : "histogram-continuous-metadata"
+                }
+                style={{
+                  padding: globals.leftSidebarSectionPadding,
+                  backgroundColor: zebra ? globals.lightestGrey : "white",
+                }}
+              >
+                <HistogramHeader
+                  fieldId={field}
+                  isColorBy={isColorAccessor}
+                  isObs={isObs}
+                  onColorByClick={this.handleColorAction(dispatch)}
+                  onRemoveClick={isUserDefined ? this.removeHistogram : null}
+                  isScatterPlotX={isScatterplotXXaccessor}
+                  isScatterPlotY={isScatterplotYYaccessor}
+                  onScatterPlotXClick={
+                    showScatterPlot ? this.handleSetGeneAsScatterplotX : null
+                  }
+                  onScatterPlotYClick={
+                    showScatterPlot ? this.handleSetGeneAsScatterplotY : null
+                  }
+                />
+                <Histogram
+                  field={field}
+                  fieldForId={fieldForId}
+                  display={asyncProps.isSingleValue ? "none" : "block"}
+                  histogram={asyncProps.histogram}
+                  width={this.width}
+                  height={this.height}
+                  onBrush={this.onBrush}
+                  onBrushEnd={this.onBrushEnd}
+                  margin={this.margin}
+                  isColorBy={isColorAccessor}
+                  selectionRange={continuousSelectionRange}
+                />
+                <HistogramFooter
+                  displayName={field}
+                  hideRanges={asyncProps.isSingleValue}
+                  rangeMin={asyncProps.unclippedRange[0]}
+                  rangeMax={asyncProps.unclippedRange[1]}
+                  rangeColorMin={asyncProps.unclippedRangeColor[0]}
+                  rangeColorMax={asyncProps.unclippedRangeColor[1]}
+                  logFoldChange={logFoldChange}
+                  pvalAdj={pvalAdj}
+                />
+              </div>
+            ) : null
+          }
+        </Async.Fulfilled>
+      </Async>
+    );
+  }
+}
+
+export default HistogramBrush;

client/src/components/categorical/annoDialog.js

@@ -0,0 +1,84 @@
+import React from "react";
+import { Button, Tooltip, Dialog, Classes, Colors } from "@blueprintjs/core";
+
+class AnnoDialog extends React.PureComponent {
+  constructor(props) {
+    super(props);
+    this.state = {};
+  }
+
+  render() {
+    const {
+      isActive,
+      text,
+      title,
+      instruction,
+      cancelTooltipContent,
+      errorMessage,
+      validationError,
+      annoSelect,
+      annoInput,
+      handleCancel,
+      handleSubmit,
+      primaryButtonText,
+      secondaryButtonText,
+      handleSecondaryButtonSubmit,
+      primaryButtonProps,
+    } = this.props;
+
+    return (
+      <Dialog icon="tag" title={title} isOpen={isActive} onClose={handleCancel}>
+        <form
+          onSubmit={(e) => {
+            e.preventDefault();
+          }}
+        >
+          <div className={Classes.DIALOG_BODY}>
+            <div style={{ marginBottom: 20 }}>
+              <p>{instruction}</p>
+              {annoInput || null}
+              <p
+                style={{
+                  marginTop: 7,
+                  visibility: validationError ? "visible" : "hidden",
+                  color: Colors.ORANGE3,
+                }}
+              >
+                {errorMessage}
+              </p>
+            </div>
+            {annoSelect || null}
+          </div>
+          <div className={Classes.DIALOG_FOOTER}>
+            <div className={Classes.DIALOG_FOOTER_ACTIONS}>
+              <Tooltip content={cancelTooltipContent}>
+                <Button onClick={handleCancel}>Cancel</Button>
+              </Tooltip>
+              {handleSecondaryButtonSubmit && secondaryButtonText ? (
+                <Button
+                  onClick={handleSecondaryButtonSubmit}
+                  disabled={!text || validationError}
+                  intent="none"
+                  type="button"
+                >
+                  {secondaryButtonText}
+                </Button>
+              ) : null}
+              <Button
+                {...primaryButtonProps} // eslint-disable-line react/jsx-props-no-spreading -- Spreading props allows for modularity
+                onClick={handleSubmit}
+                disabled={!text || validationError}
+                intent="primary"
+                type="submit"
+              >
+                {primaryButtonText}
+              </Button>
+            </div>
+          </div>
+        </form>
+      </Dialog>
+    );
+  }
+}
+
+export default AnnoDialog;

client/src/components/categorical/annoSelect.js

@@ -0,0 +1,56 @@
+import React from "react";
+import { Button, MenuItem } from "@blueprintjs/core";
+import { Select } from "@blueprintjs/select";
+
+class DuplicateCategorySelect extends React.PureComponent {
+  constructor(props) {
+    super(props);
+    this.state = {};
+  }
+
+  render() {
+    const {
+      allCategoryNames,
+      categoryToDuplicate,
+      handleModalDuplicateCategorySelection,
+    } = this.props;
+    return (
+      <div>
+        <p>
+          Optionally duplicate all labels & cell assignments from existing
+          category into new category:
+        </p>
+        <Select
+          items={
+            allCategoryNames ||
+            [] /* this is a placeholder, could be  a subcomponent to avoid this */
+          }
+          filterable={false}
+          itemRenderer={(d, { handleClick }) => {
+            return (
+              <MenuItem
+                data-testclass="duplicate-category-dropdown-option"
+                onClick={handleClick}
+                key={d}
+                text={d}
+              />
+            );
+          }}
+          noResults={<MenuItem disabled text="No results." />}
+          onItemSelect={(d) => {
+            handleModalDuplicateCategorySelection(d);
+          }}
+        >
+          {/* children become the popover target; render value here */}
+          <Button
+            data-testid="duplicate-category-dropdown"
+            text={categoryToDuplicate || "None (all cells 'unassigned')"}
+            rightIcon="double-caret-vertical"
+          />
+        </Select>
+      </div>
+    );
+  }
+}
+
+export default DuplicateCategorySelect;