bump version to 0.15.0 (#1222 )

update contributor process recommendations (#1213 )
* update contrib process recommendations * fix typos * clarification * fix build instructions * updates suggested by Matt * fix dead link
2026-09-28 12:38:11 +08:00 · 2020-03-13 13:11:12 -07:00 · 2020-03-13 11:41:30 -07:00 · 2020-03-12 20:06:44 -07:00 · 2020-03-12 15:05:08 -07:00 · 2020-03-12 08:36:36 -07:00
224 changed files with 22044 additions and 8263 deletions
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.14.1
+current_version = 0.15.0

 [bumpversion:file:setup.py]
 search = version="{current_version}"
@@ -0,0 +1,8 @@
+coverage:
+  status:
+    project:
+      default:
+        informational: true
+    patch:
+      default:
+        informational: true
@@ -0,0 +1,118 @@
+name: Compatibility Tests
+
+on:
+  schedule:
+    - cron: '0 8 7 * 2'
+  push:
+    branches:
+      - master
+
+env:
+  JEST_ENV: prod
+
+jobs:
+  docker-build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Build docker image
+        run: docker build .
+
+  cellxgene-master-with-python-and-anndata-versions:
+    name: python versions x anndata versions
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8]
+        anndata-version: [0.6.22.post1, 0.7.1]
+        test-suite: [smoke-test, smoke-test-annotations]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        # 1. only install the dev requirements on top of what is in the cellxgene pip package
+        sudo apt-get update && sudo apt-get install -y libhdf5-serial-dev
+        sed -i 's/-r requirements.txt//' server/requirements-dev.txt
+        pip install -r server/requirements-dev.txt
+        # 2. install cellxgene
+        make pydist install-dist
+        # 3. install anndata
+        pip install anndata==${{ matrix.anndata-version }}
+        # workaround for anndata 0.6.22.post1 bug
+        [[ "0.6.22.post1" = "${{ matrix.anndata-version }}" ]] && pip install h5py==2.9.0 || true
+    - name: Tests
+      run: make unit-test ${{ matrix.test-suite }}
+
+  cellxgene-release-with-anndata-master:
+    name: cellxgene release with anndata master
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        test-suite: [smoke-test, smoke-test-annotations]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          path: cellxgene
+      - name: Checkout tools repo
+        uses: actions/checkout@v2
+        with:
+          repository: theislab/anndata
+          path: anndata
+      - name: Install dependencies
+        run: |
+          cd cellxgene
+          # 1. only install the dev requirements on top of what is in the cellxgene pip package
+          make dev-env-client
+          sed -i 's/-r requirements.txt//' server/requirements-dev.txt
+          pip install -r server/requirements-dev.txt
+          # 2. install cellxgene
+          pip install --upgrade cellxgene
+          # 3. install anndata
+          cd ../anndata && pip install -e .
+      - name: Tests
+        run: cd cellxgene && make unit-test ${{ matrix.test-suite }}
+
+  cellxgene-master-with-anndata-master:
+    name: cellxgene master with anndata master
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        test-suite: [smoke-test, smoke-test-annotations]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          path: cellxgene
+      - name: Checkout tools repo
+        uses: actions/checkout@v2
+        with:
+          repository: theislab/anndata
+          path: anndata
+      - name: Install dependencies
+        run: |
+          cd cellxgene
+          sed -i -E 's/^anndata[>=]=[0-9]+.[0-9]+.[0-9]+$/anndata/g' server/requirements.txt
+          make pydist install-dist dev-env
+          cd ../anndata
+          pip install -e .
+      - name: Tests
+        run: cd cellxgene && make unit-test ${{ matrix.test-suite }}
@@ -0,0 +1,109 @@
+name: Push Tests
+
+on: [push]
+
+env:
+  JEST_ENV: prod
+  CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Lint with flake8
+      run: |
+        pip install flake8
+        make lint
+
+  unit-test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Python cache
+      uses: actions/cache@v1
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Node cache
+      uses: actions/cache@v1
+      with:
+        path: ~/.npm
+        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+        restore-keys: |
+          ${{ runner.os }}-node-
+    - name: Install dependencies
+      run: make pydist install-dist dev-env-server
+    - name: Unit tests
+      run: |
+        make unit-test
+        bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF backend,python,unitTest
+        cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
+
+  smoke-tests:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Python cache
+      uses: actions/cache@v1
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Node cache
+      uses: actions/cache@v1
+      with:
+        path: ~/.npm
+        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+        restore-keys: |
+          ${{ runner.os }}-node-
+    - name: Install dependencies
+      run: make pydist install-dist
+    - name: Smoke tests (without annotations feature)
+      run: |
+        cd client && make smoke-test
+        ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTest
+
+  smoke-tests-annotations:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Python cache
+      uses: actions/cache@v1
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Node cache
+      uses: actions/cache@v1
+      with:
+        path: ~/.npm
+        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+        restore-keys: |
+          ${{ runner.os }}-node-
+    - name: Install dependencies
+      run: make pydist install-dist
+    - name: Smoke tests (with annotations feature)
+      run: |
+        cd client && make smoke-test-annotations
+        ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTestAnnotations
@@ -3,6 +3,8 @@ node_modules/

 # coverage
 coverage
+coverage.xml
+.coverage
 .nyc_output

 .pytest_cache/
@@ -17,12 +19,12 @@ venv/
 cellxgene/

 # client build
-server/app/web/static/css/
-server/app/web/static/img/
-server/app/web/static/media/
-server/app/web/static/fonts/
-server/app/web/static/js/
-server/app/web/templates/index\.html
+server/common/web/static/css/
+server/common/web/static/img/
+server/common/web/static/media/
+server/common/web/static/fonts/
+server/common/web/static/js/
+server/common/web/templates/index\.html

 # Jupyter Notebook
 .ipynb_checkpoints
@@ -1,28 +0,0 @@
-language: python
-dist: xenial
-sudo: required
-node_js:
-  - 10
-cache:
-  - pip
-  - npm
-install:
-  - set -eo pipefail
-  - pip install flake8
-  - make pydist install-dist dev-env
-
-jobs:
-  include:
-    - name: "Branch Tests 3.7"
-      python: "3.7"
-      script: make build-client lint unit-test
-    - name: "Branch Tests 3.6"
-      python: "3.6"
-      script: make build-client lint unit-test
-    - name: "Docker Build"
-      install: skip
-      python: "3.6"
-      script: docker build .
-    - name: "Smoke Tests"
-      python: "3.6"
-      script: make smoke-test
@@ -1,5 +1,5 @@
-recursive-include server/app/web/templates *
-recursive-include server/app/web/static *
+recursive-include server/common/web/templates *
+recursive-include server/common/web/static *

 include server/requirements.txt
 include server/requirements-prepare.txt
@@ -1,3 +1,5 @@
+include common.mk
+
 BUILDDIR := build
 CLIENTBUILD := $(BUILDDIR)/client
 SERVERBUILD := $(BUILDDIR)/server
@@ -5,7 +7,6 @@ CLEANFILES :=  $(BUILDDIR)/ client/build build dist cellxgene.egg-info

 PART ?= patch

-
 # CLEANING
 .PHONY: clean
 clean: clean-lite clean-server clean-client
@@ -33,25 +34,13 @@ build-client:
 build-cli: build-client
 	git ls-files server/ | cpio -pdm $(BUILDDIR)
 	cp -r client/build/  $(CLIENTBUILD)
-	mkdir -p $(SERVERBUILD)/app/web/static/img
-	mkdir -p $(SERVERBUILD)/app/web/templates/
-	cp $(CLIENTBUILD)/index.html $(SERVERBUILD)/app/web/templates/
-	cp -r $(CLIENTBUILD)/static $(SERVERBUILD)/app/web/
-	cp $(CLIENTBUILD)/favicon.png $(SERVERBUILD)/app/web/static/img
-	cp $(CLIENTBUILD)/service-worker.js $(SERVERBUILD)/app/web/static/js/
+	$(call copy_client_assets,$(CLIENTBUILD),$(SERVERBUILD))
 	cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR)

 # If you are actively developing in the server folder use this, dirties the source tree
 .PHONY: build-for-server-dev
 build-for-server-dev: clean-server build-client
-	mkdir -p server/app/web/static/img
-	mkdir -p server/app/web/static/js
-	mkdir -p server/app/web/templates/
-	cp client/build/index.html server/app/web/templates/
-	cp -r client/build/static server/app/web/
-	cp client/build/favicon.png server/app/web/static/img
-	cp client/build/service-worker.js server/app/web/static/js/
-
+	$(call copy_client_assets,client/build,server)

 # TESTING
 .PHONY: test
@@ -67,14 +56,20 @@ unit-test-%:
 smoke-test:
 	cd client && $(MAKE) smoke-test

+.PHONY: smoke-test-annotations
+smoke-test-annotations:
+	cd client && $(MAKE) smoke-test-annotations
+
 # FORMATTING CODE

 .PHOHY: fmt
 fmt: fmt-client fmt-py

+.PHONY: fmt-client
 fmt-client:
 	cd client && $(MAKE) fmt

+.PHONY: fmt
 fmt-py:
 	black .

@@ -120,13 +115,15 @@ release-directly-to-prod: dev-env pydist twine-prod
 	@echo "    make install-release"

 .PHONY: dev-env
-dev-env:
-	cd client && $(MAKE) ci
-	pip install -r server/requirements-dev.txt
+dev-env: dev-env-client dev-env-server

-.PHONY: gui-env
-gui-env: dev-env
-	pip install -r server/requirements-gui.txt
+.PHONY: dev-env-client
+dev-env-client:
+	cd client && $(MAKE) ci
+
+.PHONY: dev-env-server
+dev-env-server:
+	pip install -r server/requirements-dev.txt

 # give PART=[major, minor, part] as param to make bump
 .PHONY: bump
@@ -182,30 +179,3 @@ install-dist: uninstall
 uninstall:
 	pip uninstall -y cellxgene || :

-
-# GUI
-
-.PHONY: build-assets
-build-assets:
-	pyside2-rcc server/gui/cellxgene.qrc -o server/gui/cellxgene_rc.py
-
-.PHONY: gui-spec-osx
-gui-spec-osx: clean-lite gui-env
-	pip install -e .[gui]
-	pyi-makespec -D -w --additional-hooks-dir server/gui/ -n cellxgene  --add-binary='/System/Library/Frameworks/Tk.framework/Tk':'tk' --add-binary='/System/Library/Frameworks/Tcl.framework/Tcl':'tcl'  --add-data server/app/web/templates/:server/app/web/templates/ --add-data server/app/web/static/:server/app/web/static/ --icon server/gui/images/cxg_icons.icns server/gui/main.py
-	mv cellxgene.spec cellxgene-osx.spec
-
-.PHONY: gui-spec-windows
-gui-spec-windows: clean-lite dev-env
-	pip install -e .[gui]
-	pyi-makespec -D -w --additional-hooks-dir server/gui/ -n cellxgene --add-data server/app/web/templates;server/app/web/templates --add-data server/app/web/static;server/app/web/static --icon server/gui/images/icon.ico server/gui/main.py
-	mv cellxgene.spec cellxgene-windows.spec
-
-.PHONY: gui-build-osx
-gui-build-osx: clean-lite
-	pyinstaller --clean cellxgene-osx.spec
-
-.PHONY: gui-build-windows
-gui-build-windows: clean-lite
-	pyinstaller --clean cellxgene-windows.spec
-
@@ -3,6 +3,9 @@
 _an interactive explorer for single-cell transcriptomics data_

 [![DOI](https://zenodo.org/badge/105615409.svg)](https://zenodo.org/badge/latestdoi/105615409) [![PyPI](https://img.shields.io/pypi/v/cellxgene)](https://pypi.org/project/cellxgene/) [![PyPI - Downloads](https://img.shields.io/pypi/dm/cellxgene)](https://pypistats.org/packages/cellxgene) [![GitHub last commit](https://img.shields.io/github/last-commit/chanzuckerberg/cellxgene)](https://github.com/chanzuckerberg/cellxgene/pulse)
+[![Push Tests](https://github.com/chanzuckerberg/cellxgene/workflows/Push%20Tests/badge.svg)](https://github.com/chanzuckerberg/cellxgene/actions?query=workflow%3A%22Push+Tests%22)
+[![Compatibility Tests](https://github.com/chanzuckerberg/cellxgene/workflows/Compatibility%20Tests/badge.svg)](https://github.com/chanzuckerberg/cellxgene/actions?query=workflow%3A%22Compatibility+Tests%22)
+![Code Coverage](https://codecov.io/gh/chanzuckerberg/cellxgene/branch/master/graph/badge.svg)

 cellxgene (pronounced "cell-by-gene") is an interactive data explorer for single-cell transcriptomics datasets, such as those coming from the [Human Cell Atlas](https://humancellatlas.org). Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data.

@@ -1,7 +1,7 @@
 {
  "name": "cellxgene",
  "description": "An interactive explorer for single-cell transcriptomics data",
-  "repository": "https://github.com/chanzuckerberg/cellxgene/",
+  "repository": "https://github.com/chanzuckerberg/cellxgene",
  "logo": "https://cellxgene-example-data.czi.technology/favicon.png",
  "keywords": [
    "scientific",
@@ -1,41 +0,0 @@
-# -*- mode: python ; coding: utf-8 -*-
-
-block_cipher = None
-
-
-a = Analysis(['server/gui/main.py'],
-             pathex=['/Users/charlotteweaver/Documents/Git/cellxgene'],
-             binaries=[('/System/Library/Frameworks/Tk.framework/Tk', 'tk'), ('/System/Library/Frameworks/Tcl.framework/Tcl', 'tcl')],
-             datas=[('server/app/web/templates/', 'server/app/web/templates/'), ('server/app/web/static/', 'server/app/web/static/')],
-             hiddenimports=['sklearn', 'sklearn.utils._cython_blas', 'sklearn.neighbors.typedefs', 'sklearn.neighbors.quad_tree', 'sklearn.tree', 'sklearn.tree._utils'],
-             hookspath=['server/gui/'],
-             runtime_hooks=[],
-             excludes=[],
-             win_no_prefer_redirects=False,
-             win_private_assemblies=False,
-             cipher=block_cipher,
-             noarchive=False)
-pyz = PYZ(a.pure, a.zipped_data,
-             cipher=block_cipher)
-exe = EXE(pyz,
-          a.scripts,
-          [],
-          exclude_binaries=True,
-          name='cellxgene',
-          debug=False,
-          bootloader_ignore_signals=False,
-          strip=False,
-          upx=True,
-          console=False , icon='server/gui/images/cxg_icons.icns')
-coll = COLLECT(exe,
-               a.binaries,
-               a.zipfiles,
-               a.datas,
-               strip=False,
-               upx=True,
-               upx_exclude=[],
-               name='cellxgene')
-app = BUNDLE(coll,
-             name='cellxgene.app',
-             icon='server/gui/images/cxg_icons.icns',
-             bundle_identifier=None)
@@ -1,36 +0,0 @@
-# -*- mode: python -*-
-
-block_cipher = None
-
-
-a = Analysis(['server\\gui\\main.py'],
-             pathex=['C:\\Users\\Charlotte\\Documents\\git\\cellxgene'],
-             binaries=[],
-             datas=[('server/app/web/templates/', 'server/app/web/templates'), ('server/app/web/static/', 'server/app/web/static')],
-             hiddenimports=[],
-             hookspath=['server/gui/'],
-             runtime_hooks=[],
-             excludes=[],
-             win_no_prefer_redirects=False,
-             win_private_assemblies=False,
-             cipher=block_cipher,
-             noarchive=False)
-pyz = PYZ(a.pure, a.zipped_data,
-             cipher=block_cipher)
-exe = EXE(pyz,
-          a.scripts,
-          [],
-          exclude_binaries=True,
-          name='cellxgene',
-          debug=False,
-          bootloader_ignore_signals=False,
-          strip=False,
-          upx=True,
-          console=False , icon='server\\gui\\images\\icon.ico')
-coll = COLLECT(exe,
-               a.binaries,
-               a.zipfiles,
-               a.datas,
-               strip=False,
-               upx=True,
-               name='cellxgene')
@@ -1,3 +1,10 @@
+include ../common.mk
+
+DATASET := $(if $(DATASET),$(DATASET),../example-dataset/pbmc3k.h5ad)
+ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../server/test/test_datasets/pbmc3k-annotations.csv)
+ANNOTATIONS_FILENAME := $(shell basename $(ANNOTATIONS))
+
+# Packaging
 .PHONY: clean
 clean:
 	rm -rf node_modules
@@ -14,7 +21,75 @@ install:
 build:
 	npm run build

+
+# Development convenience methods
+
+.PHONY: server-requirements
+server-requirements:
+	virtualenv -p python3 ../venv
+	source ../venv/bin/activate && \
+		pip install -r ../server/requirements-dev.txt && \
+		yes | pip uninstall cellxgene || true && \
+		pip install -e ..
+
+.PHONY: start-frontend
+start-frontend:
+	node server/development.js
+
+.PHONY: start-server
+start-server:
+	cellxgene launch -p $(CXG_SERVER_PORT) $(CXG_OPTIONS) $(DATASET)
+
+.PHONY: backend-dev
+backend-dev: server-requirements
+	source ../venv/bin/activate && $(MAKE) start-server
+
+.PHONY: backend-dev-anno-ontology
+backend-dev-anno-ontology: server-requirements
+	CXG_OPTIONS='--experimental-annotations-ontology' \
+		$(MAKE) backend-dev
+
+.PHONY: test
+test:
+	node node_modules/jest/bin/jest.js
+
+.PHONY: e2e
+e2e:
+	node node_modules/jest/bin/jest.js \
+		--verbose false \
+		--config __tests__/e2e/e2eJestConfig.json \
+		e2e/e2e.test.js
+
+.PHONY: e2e-annotations
+e2e-annotations:
+	node node_modules/jest/bin/jest.js \
+		--verbose false \
+		--config __tests__/e2e/e2eJestConfig.json \
+		e2e/e2eAnnotations.test.js
+
+.PHONY: smoke-test
+smoke-test:
+	start_server_and_test \
+		'CXG_OPTIONS="--disable-annotations" $(MAKE) start-server' \
+		$(CXG_SERVER_PORT) \
+		'$(MAKE) e2e'
+
+
+.PHONY: smoke-test-annotations
+smoke-test-annotations:
+	$(eval TMP_DIR := $(shell  mktemp -d /tmp/cellxgene_XXXXXX))
+	cp $(ANNOTATIONS) $(TMP_DIR)/ && \
+	start_server_and_test \
+		'CXG_OPTIONS="--annotations-file $(TMP_DIR)/$(ANNOTATIONS_FILENAME)" $(MAKE) start-server' \
+		$(CXG_SERVER_PORT) \
+		'$(MAKE) e2e-annotations'
+	rm -rf $(TMP_DIR)
+
+.PHONY: unit-test
+unit-test:
+	node node_modules/jest/bin/jest.js \
+	    --testPathIgnorePatterns e2e
+
 # pass remaining commands through to npm run
 %:
 	npm run $(*)
-
@@ -0,0 +1,196 @@
+import { strict as assert } from "assert";
+
+export const cellxgeneActions = (page, utils) => ({
+
+  async drag(testId, start, end, lasso = false) {
+    const layout = await utils.waitByID(testId);
+    const elBox = await layout.boxModel();
+    const x1 = elBox.content[0].x + start.x;
+    const x2 = elBox.content[0].x + end.x;
+    const y1 = elBox.content[0].y + start.y;
+    const y2 = elBox.content[0].y + end.y;
+    await page.mouse.move(x1, y1);
+    await page.mouse.down();
+    if (lasso) {
+      await page.mouse.move(x2, y1);
+      await page.mouse.move(x2, y2);
+      await page.mouse.move(x1, y2);
+      await page.mouse.move(x1, y1);
+    } else {
+      await page.mouse.move(x2, y2);
+    }
+    await page.mouse.up();
+  },
+
+  async clickOnCoordinate(testId, coord) {
+    const layout = await utils.waitByID(testId);
+    const elBox = await layout.boxModel();
+    const x = elBox.content[0].x + coord.x;
+    const y = elBox.content[0].y + coord.y;
+    await page.mouse.click(x, y);
+  },
+
+  async getAllHistograms(testclass, testIds) {
+    const histTestIds = testIds.map(tid => `histogram-${tid}`);
+    // these load asynchronously, so we need to wait for each histogram individually
+    await utils.waitForAllByIds(histTestIds);
+    const allHistograms = await utils.getAllByClass(testclass);
+    return allHistograms.map(hist => hist.replace(/^histogram-/, ""));
+  },
+
+  async getAllCategoriesAndCounts(category) {
+    await utils.waitByClass("categorical-row");
+    return page.$$eval(
+      `[data-testid="category-${category}"] [data-testclass='categorical-row']`,
+      rows => Object.fromEntries(rows.map(row => {
+        const cat = row.querySelector("[data-testclass='categorical-value']").innerText;
+        const count = row.querySelector("[data-testclass='categorical-value-count']").innerText;
+        return [cat, count];
+      }))
+    );
+  },
+
+  async cellSet(num) {
+    await utils.clickOn(`cellset-button-${num}`);
+    return utils.getOneElementInnerText(`[data-testid='cellset-count-${num}']`);
+  },
+
+  async resetCategory(category) {
+    const checkboxId = `${category}:category-select`;
+    await utils.waitByID(checkboxId);
+    const checkedPseudoclass = await page.$eval(
+      `[data-testid='${checkboxId}']`,
+      el => el.matches(":checked")
+    );
+    if (!checkedPseudoclass) await utils.clickOn(checkboxId);
+    try {
+      const categoryRow = await utils.waitByID(`${category}:category-expand`);
+      const isExpanded = await categoryRow.$("[data-testclass='category-expand-is-expanded']");
+      if (isExpanded) await utils.clickOn(`${category}:category-expand`);
+    } catch {}
+  },
+
+  async calcCoordinate(testId, xAsPercent, yAsPercent) {
+    const el = await utils.waitByID(testId);
+    const size = await el.boxModel();
+    return {
+      x: Math.floor(size.width * xAsPercent),
+      y: Math.floor(size.height * yAsPercent)
+    }
+  },
+
+  async calcDragCoordinates(testId, coordinateAsPercent) {
+    return {
+      start: await this.calcCoordinate(testId, coordinateAsPercent.x1, coordinateAsPercent.y1),
+      end: await this.calcCoordinate(testId, coordinateAsPercent.x2, coordinateAsPercent.y2)
+    };
+  },
+
+  async selectCategory(category, values, reset = true) {
+    if (reset) await this.resetCategory(category);
+    await utils.clickOn(`${category}:category-expand`);
+    await utils.clickOn(`${category}:category-select`);
+    for (const val of values) {
+      await utils.clickOn(`categorical-value-select-${category}-${val}`);
+    }
+  },
+
+  async expandCategory(category) {
+    const expand = await utils.waitByID(`${category}:category-expand`);
+    const notExpanded = await expand.$("[data-testclass='category-expand-is-not-expanded']");
+    if (notExpanded) await utils.clickOn(`${category}:category-expand`);
+  },
+
+  async clip(min = 0, max = 100) {
+    await utils.clickOn("visualization-settings");
+    await utils.clearInputAndTypeInto("clip-min-input", min);
+    await utils.clearInputAndTypeInto("clip-max-input", max);
+    await utils.clickOn("clip-commit");
+  },
+
+  async createCategory(categoryName) {
+    await utils.clickOn("open-annotation-dialog");
+    await utils.typeInto("new-category-name", categoryName);
+    await utils.clickOn("submit-category");
+  },
+
+  async renameCategory(oldCatgoryName, newCategoryName) {
+    await utils.clickOn(`${oldCatgoryName}:see-actions`);
+    await utils.clickOn(`${oldCatgoryName}:edit-category-mode`);
+    await utils.clearInputAndTypeInto(`${oldCatgoryName}:edit-category-name-text`, newCategoryName);
+    await utils.clickOn(`${oldCatgoryName}:submit-category-edit`);
+  },
+
+  async deleteCategory(categoryName) {
+    await utils.clickOn(`${categoryName}:see-actions`);
+    await utils.clickOn(`${categoryName}:delete-category`);
+  },
+
+  async createLabel(categoryName, labelName) {
+    await utils.clickOn(`${categoryName}:see-actions`);
+    await utils.clickOn(`${categoryName}:add-new-label-to-category`);
+    await utils.typeInto(`${categoryName}:new-label-name`, labelName);
+    await utils.clickOn(`${categoryName}:submit-label`);
+  },
+
+  async deleteLabel(categoryName, labelName) {
+    await this.expandCategory(categoryName);
+    await utils.clickOn(`${categoryName}:${labelName}:see-actions`);
+    await utils.clickOn( `${categoryName}:${labelName}:delete-label`);
+  },
+
+  async renameLabel(categoryName, oldLabelName, newLabelName) {
+    await this.expandCategory(categoryName);
+    await utils.clickOn(`${categoryName}:${oldLabelName}:see-actions`);
+    await utils.clickOn(`${categoryName}:${oldLabelName}:edit-label`);
+    await utils.clearInputAndTypeInto(
+      `${categoryName}:${oldLabelName}:edit-label-name`,
+      newLabelName
+    );
+    await utils.clickOn(`${categoryName}:${oldLabelName}:submit-label-edit`);
+  },
+
+  async addGeneToSearch(geneName) {
+    await utils.typeInto("gene-search", geneName);
+    await page.keyboard.press("Enter");
+    await page.waitForSelector(
+      `[data-testid='histogram-${geneName}']`
+    );
+  },
+
+  async subset(coordinatesAsPercent) {
+    // In order to deselect the selection after the subset, make sure we have some clear part
+    // of the scatterplot we can click on
+    assert(coordinatesAsPercent.x2 < 0.99 || coordinatesAsPercent.y2 < 0.99);
+    const lassoSelection = await this.calcDragCoordinates( "layout-graph", coordinatesAsPercent);
+    await this.drag("layout-graph", lassoSelection.start, lassoSelection.end, true );
+    await utils.clickOn("subset-button");
+    const clearCoordinate = await this.calcCoordinate(
+      "layout-graph",
+      0.5,
+      0.99
+    );
+    await this.clickOnCoordinate("layout-graph", clearCoordinate);
+  },
+
+  async setSellSet(cellSet, cellSetNum) {
+    for (const selection of cellSet.filter(sel => sel.kind === "categorical")) {
+      await this.selectCategory(selection.metadata, selection.values, true);
+    }
+    await this.cellSet(cellSetNum);
+  },
+
+  async runDiffExp(cellSet1, cellSet2) {
+    await this.setSellSet(cellSet1, 1);
+    await this.setSellSet(cellSet2, 2);
+    await utils.clickOn("diffexp-button");
+  },
+
+  async bulkAddGenes(geneNames) {
+    await utils.clickOn("section-bulk-add");
+    await utils.typeInto("input-bulk-add", geneNames.join(","));
+    await page.keyboard.press("Enter");
+  }
+});
+
+
@@ -1,6 +1,9 @@
-export const jest_env = process.env.JEST_ENV || "dev";
-export const appPort = process.env.JEST_CXG_PORT || 3000;
+export const jest_env = process.env.JEST_ENV;
+export const appPort = process.env.CXG_SERVER_PORT;
 export const appUrlBase = `http://localhost:${appPort}`;
 export const DEV = jest_env === "dev";
 export const DEBUG = jest_env === "debug";
 export const DATASET = "pbmc3k";
+
+if (DEBUG) jest.setTimeout(100000);
+if (DEV) jest.setTimeout(10000);
@@ -115,6 +115,25 @@ export const datasets = {
        }
      }
    },
+    categoryLabel: {
+      lasso: {
+        "coordinates-as-percent": { x1: 0.05, y1: 0.3, x2: 0.5, y2: 0.5 },
+      },
+      newCount: {
+        bySubsetConfig: {
+          false: "600",
+          true: "591"
+        }
+      }
+    },
+    annotationsFromFile: {
+      count: {
+        bySubsetConfig: {
+          false: "1161",
+          true: "856"
+        }
+      }
+    },
    clip: {
      min: "30",
      max: "70",
@@ -3,45 +3,16 @@ Smoke test suite that will be run in Travis CI

 Tests included in this file are expected to be relatively stable and test core features
 */
-import puppeteer from "puppeteer";
-import { appUrlBase, DEBUG, DEV, DATASET } from "./config";
-import { puppeteerUtils, cellxgeneActions } from "./puppeteerUtils";
+import { appUrlBase, DATASET } from "./config";
+import { setupTestBrowser } from "./testBrowser";
 import { datasets } from "./data";

-let browser, page, utils, cxgActions, spy;
-const browserViewport = { width: 1280, height: 960 };
-let data = datasets[DATASET];
-
-if (DEBUG) jest.setTimeout(100000);
-if (DEV) jest.setTimeout(10000);
+let browser, page, utils, cxgActions;
+const data = datasets[DATASET];

 beforeAll(async () => {
-  const browserParams = DEV
-    ? { headless: false, slowMo: 5 }
-    : DEBUG
-    ? { headless: false, slowMo: 100, devtools: true }
-    : {};
-  browser = await puppeteer.launch(browserParams);
-  page = await browser.newPage();
-  await page.setViewport(browserViewport);
-  if (DEV || DEBUG) {
-    page.on("console", async msg => {
-      // If there is a console.error but an error is not thrown, this will ensure the test fails
-      if (msg.type() === "error") {
-        const errorMsgText = await Promise.all(
-          // TODO can we do this without internal properties?
-          msg.args().map(arg => arg._remoteObject.description)
-        );
-        throw new Error(`Console error: ${errorMsgText}`);
-      }
-      console.log(`PAGE LOG: ${msg.text()}`);
-    });
-  }
-  page.on("pageerror", err => {
-    throw new Error(`Console error: ${err}`);
-  });
-  utils = puppeteerUtils(page);
-  cxgActions = cellxgeneActions(page);
+  const browserViewport = { width: 1280, height: 960 };
+  [browser, page, utils, cxgActions] = await setupTestBrowser(browserViewport);
 });

 beforeEach(async () => {
@@ -49,15 +20,13 @@ beforeEach(async () => {
 });

 afterAll(() => {
-  if (!DEBUG) {
-    browser.close();
-  }
+  if (browser !== undefined) browser.close();
 });

 describe("did launch", () => {
  test("page launched", async () => {
-    let el = await utils.getOneElementInnerHTML("[data-testid='header']");
-    expect(el).toBe(data.title);
+    const element = await utils.getOneElementInnerHTML("[data-testid='header']");
+    expect(element).toBe(data.title);
  });
 });

@@ -65,11 +34,9 @@ describe("metadata loads", () => {
  test("categories and values from dataset appear", async () => {
    for (const label in data.categorical) {
      await utils.waitByID(`category-${label}`);
-      const categoryName = await utils.getOneElementInnerText(
-        `[data-testid="category-${label}"]`
-      );
+      const categoryName = await utils.getOneElementInnerText(`[data-testid="category-${label}"]`);
      expect(categoryName).toMatch(label);
-      await utils.clickOn(`category-expand-${label}`);
+      await utils.clickOn(`${label}:category-expand`);
      const categories = await cxgActions.getAllCategoriesAndCounts(label);
      expect(Object.keys(categories)).toMatchObject(
        Object.keys(data.categorical[label])
@@ -112,12 +79,10 @@ describe("cell selection", () => {

  test("selects cells via categorical", async () => {
    for (const cellset of data.cellsets.categorical) {
-      await utils.clickOn(`category-expand-${cellset.metadata}`);
-      await utils.clickOn(`category-select-${cellset.metadata}`);
+      await utils.clickOn(`${cellset.metadata}:category-expand`);
+      await utils.clickOn(`${cellset.metadata}:category-select`);
      for (const val of cellset.values) {
-        await utils.clickOn(
-          `categorical-value-select-${cellset.metadata}-${val}`
-        );
+        await utils.clickOn(`categorical-value-select-${cellset.metadata}-${val}`);
      }
      const cellCount = await cxgActions.cellSet(1);
      expect(cellCount).toBe(cellset.count);
@@ -139,46 +104,20 @@ describe("cell selection", () => {
 });

 describe("gene entry", () => {
-  test("search for single gene", async () => {
-    // blueprint's  typeahead is treating typing weird, clicking & waiting first solves this
-    await utils.typeInto("gene-search", data.genes.search);
-    await page.keyboard.press("Enter");
-    await page.waitForSelector(
-      `[data-testid='histogram-${data.genes.search}']`
-    );
-  });
+  test("search for single gene", async () => cxgActions.addGeneToSearch(data.genes.search));

  test("bulk add genes", async () => {
-    await cxgActions.reset();
    const testGenes = data.genes.bulkadd;
-    await utils.clickOn("section-bulk-add");
-    await utils.typeInto("input-bulk-add", testGenes.join(","));
-    await page.keyboard.press("Enter");
-
-    const allHistograms = await cxgActions.getAllHistograms(
-      "histogram-user-gene",
-      testGenes
-    );
+    await cxgActions.bulkAddGenes(testGenes);
+    const allHistograms = await cxgActions.getAllHistograms("histogram-user-gene", testGenes);
    expect(allHistograms).toEqual(expect.arrayContaining(testGenes));
    expect(allHistograms.length).toEqual(testGenes.length);
  });
 });

-describe("diffexp", () => {
+describe("differential expression", () => {
  test("selects cells, saves them and performs diffexp", async () => {
-    for (const select of data.diffexp.cellset1) {
-      if (select.kind === "categorical") {
-        await cxgActions.selectCategory(select.metadata, select.values, true);
-      }
-    }
-    await cxgActions.cellSet(1);
-    for (const select of data.diffexp.cellset2) {
-      if (select.kind === "categorical") {
-        await cxgActions.selectCategory(select.metadata, select.values, true);
-      }
-    }
-    await cxgActions.cellSet(2);
-    await utils.clickOn("diffexp-button");
+    await cxgActions.runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
    const allHistograms = await cxgActions.getAllHistograms(
      "histogram-diffexp",
      data.diffexp["gene-results"]
@@ -190,7 +129,7 @@ describe("diffexp", () => {
  });
 });

-describe("subset/reset", () => {
+describe("subset", () => {
  test("subset - cell count matches", async () => {
    for (const select of data.subset.cellset1) {
      if (select.kind === "categorical") {
@@ -209,39 +148,6 @@ describe("subset/reset", () => {
    }
  });

-  test("reset after subset", async () => {
-    for (const select of data.subset.cellset1) {
-      if (select.kind === "categorical") {
-        await cxgActions.selectCategory(select.metadata, select.values, true);
-      }
-    }
-    await utils.clickOn("subset-button");
-    for (const label in data.subset.categorical) {
-      const categories = await cxgActions.getAllCategoriesAndCounts(label);
-      expect(Object.keys(categories)).toMatchObject(
-        Object.keys(data.subset.categorical[label])
-      );
-      expect(Object.values(categories)).toMatchObject(
-        Object.values(data.subset.categorical[label])
-      );
-    }
-    await cxgActions.reset();
-    for (const label in data.categorical) {
-      await utils.waitByID(`category-${label}`);
-      const categoryName = await utils.getOneElementInnerText(
-        `[data-testid="category-${label}"]`
-      );
-      expect(categoryName).toMatch(label);
-      const categories = await cxgActions.getAllCategoriesAndCounts(label);
-      expect(Object.keys(categories)).toMatchObject(
-        Object.keys(data.categorical[label])
-      );
-      expect(Object.values(categories)).toMatchObject(
-        Object.values(data.categorical[label])
-      );
-    }
-  });
-
  test("lasso after subset", async () => {
    for (const select of data.subset.cellset1) {
      if (select.kind === "categorical") {
@@ -262,15 +168,49 @@ describe("subset/reset", () => {
    const cellCount = await cxgActions.cellSet(1);
    expect(cellCount).toBe(data.subset.lasso.count);
  });
+
+  test("undo selection appends the top diff exp genes to user defined genes", async () => {
+    const userDefinedGenes = data.genes.bulkadd;
+    const diffExpGenes = data.diffexp["gene-results"];
+    await cxgActions.bulkAddGenes(userDefinedGenes);
+    const userDefinedHistograms = await cxgActions.getAllHistograms("histogram-user-gene", userDefinedGenes);
+    expect(userDefinedHistograms).toEqual(expect.arrayContaining(userDefinedGenes));
+    await cxgActions.subset({x1: 0.15, y1: 0.10, x2: 0.98, y2: 0.98});
+    await cxgActions.runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
+    const diffExpHistograms = await cxgActions.getAllHistograms("histogram-diffexp", diffExpGenes);
+    expect(diffExpHistograms).toEqual(expect.arrayContaining(diffExpGenes));
+    await utils.clickOn("reset-subset-button");
+    const expected = [].concat(userDefinedGenes, diffExpGenes);
+    const userDefinedHistogramsAfterSubset = await cxgActions.getAllHistograms(
+      "histogram-user-gene",
+      expected
+    );
+    expect(userDefinedHistogramsAfterSubset).toEqual(expect.arrayContaining(expected));
+  });
+
+  test("subset selection appends the top diff exp genes to user defined genes", async () => {
+    const userDefinedGenes = data.genes.bulkadd;
+    const diffExpGenes = data.diffexp["gene-results"];
+    await cxgActions.bulkAddGenes(userDefinedGenes);
+    const userDefinedHistograms = await cxgActions.getAllHistograms("histogram-user-gene", userDefinedGenes);
+    expect(userDefinedHistograms).toEqual(expect.arrayContaining(userDefinedGenes));
+    await cxgActions.subset({x1: 0.15, y1: 0.10, x2: 0.98, y2: 0.98});
+    await cxgActions.runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
+    const diffExpHistograms = await cxgActions.getAllHistograms("histogram-diffexp", diffExpGenes);
+    expect(diffExpHistograms).toEqual(expect.arrayContaining(diffExpGenes));
+    await cxgActions.subset({x1: 0.16, y1: 0.11, x2: 0.97, y2: 0.97});
+    const expected = [].concat(userDefinedGenes, diffExpGenes);
+    const userDefinedHistogramsAfterSubset = await cxgActions.getAllHistograms(
+      "histogram-user-gene",
+      expected
+    );
+    expect(userDefinedHistogramsAfterSubset).toEqual(expect.arrayContaining(expected));
+  });
 });

 describe("scatter plot", () => {
  test("scatter plot appears", async () => {
-    await cxgActions.reset();
-    const testGenes = data.scatter.genes;
-    await utils.clickOn("section-bulk-add");
-    await utils.typeInto("input-bulk-add", Object.values(testGenes).join(","));
-    await page.keyboard.press("Enter");
+    await cxgActions.bulkAddGenes(Object.values(data.scatter.genes));
    await utils.clickOn(`plot-x-${data.scatter.genes.x}`);
    await utils.clickOn(`plot-y-${data.scatter.genes.y}`);
    await utils.waitByID("scatterplot");
@@ -338,6 +278,58 @@ describe("ui elements don't error", () => {
      panCoords.end,
      false
    );
-    await page.evaluate(`window.scrollBy(0, 1000);`);
+    await page.evaluate("window.scrollBy(0, 1000);");
+  });
+});
+
+describe("centroid labels", () => {
+  test("labels are created", async () => {
+    await utils.clickOn("centroid-label-toggle");
+    const labels = Object.keys(data.categorical);
+    /* eslint-disable no-await-in-loop */
+    // Toggle colorby for each category and check to see if labels are generated
+    for (let i = 0, { length } = labels; i < length; i += 1) {
+      const label = labels[i];
+      await utils.clickOn(`colorby-${label}`);
+      const generatedLabels = await utils.getAllByClass("centroid-label");
+      // Number of labels generated should be equal to size of the object
+      expect(generatedLabels).toHaveLength(
+        Object.keys(data.categorical[label]).length
+      );
+    }
+    /* eslint-enable no-await-in-loop */
+  });
+});
+
+describe("graph overlay", () => {
+  test("transform centroids correctly", async () => {
+    const category = Object.keys(data.categorical)[0];
+    await utils.clickOn("centroid-label-toggle");
+    await utils.clickOn(`colorby-${category}`);
+    await utils.clickOn("mode-pan-zoom");
+    const panCoords = await cxgActions.calcDragCoordinates(
+      "layout-graph",
+      data.pan["coordinates-as-percent"]
+    );
+
+    const categoryValue = Object.keys(data.categorical[category])[0];
+    const initialCoordinates = await utils.getElementCoordinates(
+      `${categoryValue}-centroid-label`
+    );
+    await cxgActions.drag(
+      "layout-graph",
+      panCoords.start,
+      panCoords.end,
+      false
+    );
+    const terminalCoordinates = await utils.getElementCoordinates(
+      `${categoryValue}-centroid-label`
+    );
+    expect(terminalCoordinates[0] - initialCoordinates[0]).toBeCloseTo(
+      panCoords.end.x - panCoords.start.x
+    );
+    expect(terminalCoordinates[1] - initialCoordinates[1]).toBeCloseTo(
+      panCoords.end.y - panCoords.start.y
+    );
  });
 });
@@ -0,0 +1,222 @@
+/*
+Tests included in this file are specific to annotation features
+*/
+import { appUrlBase, DATASET } from "./config";
+import { setupTestBrowser } from "./testBrowser";
+import { datasets } from "./data";
+
+let browser, page, utils, actions;
+const data = datasets[DATASET];
+
+beforeAll(async () => {
+  const browserViewport = {width: 1280, height: 960};
+  [browser, page, utils, actions] = await setupTestBrowser(browserViewport);
+});
+
+afterAll(() => {
+  if (browser !== undefined) browser.close()
+});
+
+describe.each([
+  {withSubset: true, tag: "subset"},
+  {withSubset: false, tag: "whole"}
+])("annotations", (config) => {
+
+  const perTestCategoryName = "per-test-category";
+  const perTestLabelName = "per-test-label";
+
+  beforeEach(async () => {
+    await page.goto(appUrlBase);
+    // wait for the page to load
+    await utils.waitByClass("autosave-complete");
+    // setup the test fixtures
+    await actions.createCategory(perTestCategoryName);
+    await actions.createLabel(perTestCategoryName, perTestLabelName);
+    if (config.withSubset) await actions.subset({x1: 0.10, y1: 0.10, x2: 0.80, y2: 0.80});
+    await utils.waitByClass("autosave-complete");
+  });
+
+  afterEach(async () => {
+    await deleteCategoryIfExists(perTestCategoryName);
+    await utils.waitByClass("autosave-complete");
+  });
+
+  test("create a category", async () => {
+    const categoryName = `category-created-${config.tag}`;
+    await assertCategoryDoesNotExist(categoryName);
+    await actions.createCategory(categoryName);
+    await assertCategoryExists(categoryName);
+  });
+
+  test("delete a category", async () => {
+    await actions.deleteCategory(perTestCategoryName);
+    await assertCategoryDoesNotExist(perTestCategoryName);
+  });
+
+  test("rename a category", async () => {
+    const newCategoryName = `cluster-for-real-${config.tag}`;
+    await actions.renameCategory(perTestCategoryName, newCategoryName);
+    await assertCategoryDoesNotExist(perTestCategoryName);
+    await assertCategoryExists(newCategoryName);
+  });
+
+  test("create a label", async () => {
+    const labelName = `new-label-${config.tag}`;
+    await assertLabelDoesNotExist(perTestCategoryName, labelName);
+    await actions.createLabel(perTestCategoryName, labelName);
+    await assertLabelExists(perTestCategoryName, labelName);
+  });
+
+  test("delete a label", async () => {
+    await actions.deleteLabel(perTestCategoryName, perTestLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName, perTestLabelName);
+  });
+
+  test("rename a label", async () => {
+    const newLabelName = "my-cool-new-label";
+    await assertLabelDoesNotExist(perTestCategoryName, newLabelName);
+    await actions.renameLabel(perTestCategoryName, perTestLabelName, newLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName, perTestLabelName);
+    await assertLabelExists(perTestCategoryName, newLabelName);
+  });
+
+  test("check cell count for a label loaded from file", async () => {
+    const categoryName = "cluster-test";
+    const labelName = "four";
+    await actions.expandCategory(categoryName);
+    const result = await utils.waitByID(`categorical-value-count-${categoryName}-${labelName}`);
+    expect(await result.evaluate(node => node.innerText)).toBe(
+      data.annotationsFromFile.count.bySubsetConfig[config.withSubset]
+    );
+  });
+
+  test("assign cells to a label", async () => {
+    await actions.expandCategory(perTestCategoryName);
+    const lassoSelection = await actions.calcDragCoordinates(
+      "layout-graph",
+      data.categoryLabel.lasso["coordinates-as-percent"]
+    );
+    await actions.drag(
+      "layout-graph",
+      lassoSelection.start,
+      lassoSelection.end,
+      true
+    );
+    await utils.waitByID("lasso-element", {visible: true});
+    await utils.clickOn(`${perTestCategoryName}:${perTestLabelName}:see-actions`);
+    await utils.clickOn(`${perTestCategoryName}:${perTestLabelName}:add-current-selection-to-this-label`);
+    const result = await utils.waitByID(`categorical-value-count-${perTestCategoryName}-${perTestLabelName}`);
+    expect(await result.evaluate(node => node.innerText)).toBe(
+      data.categoryLabel.newCount.bySubsetConfig[config.withSubset]
+    );
+  });
+
+  test("undo/redo category creation", async () => {
+    const categoryName = `category-created-undo-${config.tag}`;
+    await assertCategoryDoesNotExist(categoryName);
+    await actions.createCategory(categoryName);
+    await assertCategoryExists(categoryName);
+    await utils.clickOn("undo");
+    await assertCategoryDoesNotExist(categoryName);
+    await utils.clickOn("redo");
+    await assertCategoryExists(categoryName);
+  });
+
+  test("undo/redo category deletion", async () => {
+    const categoryName = `category-deleted-undo-${config.tag}`;
+    await actions.createCategory(categoryName);
+    await assertCategoryExists(categoryName);
+    await actions.deleteCategory(categoryName);
+    await assertCategoryDoesNotExist(categoryName);
+    await utils.clickOn("undo");
+    await assertCategoryExists(categoryName);
+    await utils.clickOn("redo");
+    await assertCategoryDoesNotExist(categoryName);
+  });
+
+  test("undo/redo category rename", async () => {
+    const newCategoryName = `category-renamed-undo-${config.tag}`;
+    await assertCategoryDoesNotExist(newCategoryName);
+    await actions.renameCategory(perTestCategoryName, newCategoryName);
+    await assertCategoryExists(newCategoryName);
+    await assertCategoryDoesNotExist(perTestCategoryName);
+    await utils.clickOn("undo");
+    await assertCategoryExists(perTestCategoryName);
+    await assertCategoryDoesNotExist(newCategoryName);
+    await utils.clickOn("redo");
+    await assertCategoryExists(newCategoryName);
+    await assertCategoryDoesNotExist(perTestCategoryName);
+  });
+
+  test("undo/redo label creation", async () => {
+    const labelName = `label-created-undo-${config.tag}`;
+    await assertLabelDoesNotExist(perTestCategoryName, labelName);
+    await actions.createLabel(perTestCategoryName, labelName);
+    await assertLabelExists(perTestCategoryName, labelName);
+    await utils.clickOn("undo");
+    await assertLabelDoesNotExist(perTestCategoryName);
+    await utils.clickOn("redo");
+    await assertLabelExists(perTestCategoryName, labelName);
+  });
+
+  test("undo/redo label deletion", async () => {
+    await actions.deleteLabel(perTestCategoryName, perTestLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName);
+    await utils.clickOn("undo");
+    await assertLabelExists(perTestCategoryName, perTestLabelName);
+    await utils.clickOn("redo");
+    await assertLabelDoesNotExist(perTestCategoryName);
+  });
+
+  test("undo/redo label rename", async () => {
+    const newLabelName = `label-renamed-undo-${config.tag}`;
+    await assertLabelDoesNotExist(perTestCategoryName, newLabelName);
+    await actions.renameLabel(perTestCategoryName, perTestLabelName, newLabelName);
+    await assertLabelExists(perTestCategoryName, newLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName, perTestLabelName);
+    await utils.clickOn("undo");
+    await assertLabelExists(perTestCategoryName, perTestLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName, newLabelName);
+    await utils.clickOn("redo");
+    await assertLabelExists(perTestCategoryName, newLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName, perTestLabelName);
+  });
+
+  async function assertCategoryExists(categoryName) {
+    const handle = await utils.waitByID(`${categoryName}:category-expand`);
+    const result = await handle.evaluate(node => node.innerText);
+    // slice beginning and end of category name result to account for truncation of long names
+    expect(result.slice(0, 10)).toBe(categoryName.slice(0, 10));
+    expect(result.slice(-10)).toBe(categoryName.slice(-10));
+  }
+
+  async function assertCategoryDoesNotExist(categoryName) {
+    const result = await page.$(`[data-testid='${categoryName}:category-expand']`);
+    expect(result).toBeNull();
+  }
+
+  async function assertLabelExists(categoryName, labelName) {
+    const category = await utils.waitByID(`${categoryName}:category-expand`);
+    expect(category).not.toBeNull();
+    await actions.expandCategory(categoryName);
+    const previous = await utils.waitByID(`categorical-value-${categoryName}-${labelName}`);
+    expect(await previous.evaluate(node => node.innerText)).toBe(labelName);
+  }
+
+  async function assertLabelDoesNotExist(categoryName, labelName) {
+    await actions.expandCategory(categoryName);
+    const result = await page.$(`[data-testid='categorical-value-${categoryName}-${labelName}']`);
+    expect(result).toBeNull();
+  }
+
+  async function deleteCategoryIfExists(categoryName) {
+    try {
+      const category = await page.waitForSelector(
+        `[data-testid='${categoryName}:category-expand']`,
+        {timeout: 200}
+      );
+      if (category !== null) return await actions.deleteCategory(categoryName);
+    } catch {}
+    return null
+  }
+});
@@ -1,211 +1,69 @@
-export const puppeteerUtils = puppeteerPage => ({
-  async waitByID(testid, props = {}) {
-    return await puppeteerPage.waitForSelector(
-      `[data-testid='${testid}']`,
-      props
-    );
+export const puppeteerUtils = page => ({
+  async waitByID(testId, props = {}) {
+    return page.waitForSelector(`[data-testid='${testId}']`, props);
  },

-  async waitByClass(testclass, props = {}) {
-    return await puppeteerPage.waitForSelector(
-      `[data-testclass='${testclass}']`,
-      props
-    );
+  async waitByClass(testClass, props = {}) {
+    return page.waitForSelector(`[data-testclass='${testClass}']`, props);
  },

-  async waitForAllByIds(testids, props = {}) {
+  async waitForAllByIds(testIds) {
    await Promise.all(
-      testids.map(testid =>
-        puppeteerPage.waitForSelector(`[data-testid='${testid}']`)
-      )
+      testIds.map(testId => page.waitForSelector(`[data-testid='${testId}']`))
    );
  },

-  async getAllByClass(testclass, props = {}) {
-    const elements = await puppeteerPage.$$eval(
-      `[data-testclass=${testclass}]`,
-      els => {
-        return els.map(el => {
-          return el.dataset.testid;
-        });
-      }
+  async getAllByClass(testClass) {
+    return page.$$eval(`[data-testclass=${testClass}]`, eles =>
+      eles.map(ele => ele.dataset.testid)
    );
-    return elements;
  },

-  async typeInto(testid, text) {
+  async typeInto(testId, text) {
+    // blueprint's  typeahead is treating typing weird, clicking & waiting first solves this
    // only works for text without special characters
-    await this.waitByID(testid);
-    const selector = `[data-testid='${testid}']`;
+    await this.waitByID(testId);
+    const selector = `[data-testid='${testId}']`;
    // type ahead can be annoying if you don't pause before you type
-    await puppeteerPage.click(selector);
-    await puppeteerPage.waitFor(200);
-    await puppeteerPage.type(selector, text);
+    await page.click(selector);
+    await page.waitFor(200);
+    await page.type(selector, text);
  },

-  async clearInputAndTypeInto(testid, text) {
-    await this.waitByID(testid);
-    const selector = `[data-testid='${testid}']`;
+  async clearInputAndTypeInto(testId, text) {
+    await this.waitByID(testId);
+    const selector = `[data-testid='${testId}']`;
    // only works for text without special characters
    // type ahead can be annoying if you don't pause before you type
-    await puppeteerPage.click(selector);
-    await puppeteerPage.waitFor(200);
+    await page.click(selector);
+    await page.waitFor(200);
    // select all
-
-    await puppeteerPage.click(selector, { clickCount: 3 });
-    await puppeteerPage.keyboard.type("Backspace");
-    await puppeteerPage.type(selector, text);
+    await page.click(selector, { clickCount: 3 });
+    await page.keyboard.press("Backspace");
+    await page.type(selector, text);
  },

-  async clickOn(testid) {
+  async clickOn(testid, options = {}) {
    await this.waitByID(testid);
-    await puppeteerPage.click(`[data-testid='${testid}']`);
-    await puppeteerPage.waitFor(50);
+    const click = await page.click(`[data-testid='${testid}']`, options);
+    await page.waitFor(50);
+    return click;
  },

  async getOneElementInnerHTML(selector) {
-    await puppeteerPage.waitForSelector(selector);
-    let text = await puppeteerPage.$eval(selector, el => el.innerHTML);
-    return text;
+    await page.waitForSelector(selector);
+    return page.$eval(selector, el => el.innerHTML);
  },

  async getOneElementInnerText(selector) {
-    await puppeteerPage.waitForSelector(selector);
-    let text = await puppeteerPage.$eval(selector, el => el.innerText);
-    return text;
-  }
-});
-
-export const cellxgeneActions = puppeteerPage => ({
-  async drag(testid, start, end, lasso = false) {
-    const layout = await puppeteerUtils(puppeteerPage).waitByID(testid);
-    const elBox = await layout.boxModel();
-    const x1 = elBox.content[0].x + start.x;
-    const x2 = elBox.content[0].x + end.x;
-    const y1 = elBox.content[0].y + start.y;
-    const y2 = elBox.content[0].y + end.y;
-    await puppeteerPage.mouse.move(x1, y1);
-    await puppeteerPage.mouse.down();
-    if (lasso) {
-      await puppeteerPage.mouse.move(x2, y1);
-      await puppeteerPage.mouse.move(x2, y2);
-      await puppeteerPage.mouse.move(x1, y2);
-      await puppeteerPage.mouse.move(x1, y1);
-    } else {
-      await puppeteerPage.mouse.move(x2, y2);
-    }
-    await puppeteerPage.mouse.up();
-  },
-
-  async getAllHistograms(testclass, testids) {
-    const histTestIds = testids.map(tid => `histogram-${tid}`);
-    // these load asynchronously, so we need to wait for each histogram individually
-    await puppeteerUtils(puppeteerPage).waitForAllByIds(histTestIds);
-    const allHistograms = await puppeteerUtils(puppeteerPage).getAllByClass(
-      testclass
-    );
-    return allHistograms.map(hist =>
-      hist.substr("histogram_".length, hist.length)
-    );
-  },
-
-  async getAllCategoriesAndCounts(category) {
-    await puppeteerUtils(puppeteerPage).waitByClass("categorical-row");
-    const categories = await puppeteerPage.$$eval(
-      `[data-testid="category-${category}"] [data-testclass='categorical-row']`,
-      els => {
-        let result = {};
-        els.forEach(el => {
-          const cat = el.querySelector("[data-testclass='categorical-value']")
-            .innerText;
-          const count = el.querySelector(
-            "[data-testclass='categorical-value-count']"
-          ).innerText;
-          result[cat] = count;
-        });
-        return result;
-      }
-    );
-    return categories;
-  },
-
-  async cellSet(num) {
-    await puppeteerUtils(puppeteerPage).clickOn(`cellset-button-${num}`);
-    return await puppeteerUtils(puppeteerPage).getOneElementInnerText(
-      `[data-testid='cellset-count-${num}']`
-    );
-  },
-
-  async resetCategory(category) {
-    const checkboxId = `category-select-${category}`;
-    await puppeteerUtils(puppeteerPage).waitByID(checkboxId);
-    const checkedPseudoclass = await puppeteerPage.$eval(
-      `[data-testid='${checkboxId}']`,
-      el => {
-        return el.matches(":checked");
-      }
-    );
-    if (!checkedPseudoclass) {
-      await puppeteerUtils(puppeteerPage).clickOn(checkboxId);
-    }
-    try {
-      const categoryRow = await puppeteerUtils(puppeteerPage).waitByID(
-        `category-expand-${category}`
-      );
-      const isExpanded = await categoryRow.$(
-        "[data-testclass='category-expand-is-expanded']"
-      );
-      if (isExpanded) {
-        await puppeteerUtils(puppeteerPage).clickOn(
-          `category-expand-${category}`
-        );
-      }
-    } catch {}
-  },
-
-  async calcDragCoordinates(testid, coordinateAsPercent) {
-    const el = await puppeteerUtils(puppeteerPage).waitByID(testid);
-    const size = await el.boxModel();
-    const coords = {
-      start: {
-        x: Math.floor(size.width * coordinateAsPercent.x1),
-        y: Math.floor(size.height * coordinateAsPercent.y1)
-      },
-      end: {
-        x: Math.floor(size.width * coordinateAsPercent.x2),
-        y: Math.floor(size.height * coordinateAsPercent.y2)
-      }
-    };
-    return coords;
-  },
-
-  async selectCategory(category, values, reset = true) {
-    if (reset) await this.resetCategory(category);
-    await puppeteerUtils(puppeteerPage).clickOn(`category-expand-${category}`);
-    await puppeteerUtils(puppeteerPage).clickOn(`category-select-${category}`);
-    for (const val of values) {
-      await puppeteerUtils(puppeteerPage).clickOn(
-        `categorical-value-select-${category}-${val}`
-      );
-    }
-  },
-
-  async reset() {
-    await puppeteerUtils(puppeteerPage).clickOn("reset");
-    // loading state never actually happens, reset is too fast
-    await page.waitFor(200);
+    await page.waitForSelector(selector);
+    return page.$eval(selector, el => el.innerText);
  },

-  async clip(min = 0, max = 100) {
-    await puppeteerUtils(puppeteerPage).clickOn("visualization-settings");
-    await puppeteerUtils(puppeteerPage).clearInputAndTypeInto(
-      "clip-min-input",
-      min
-    );
-    await puppeteerUtils(puppeteerPage).clearInputAndTypeInto(
-      "clip-max-input",
-      max
-    );
-    await puppeteerUtils(puppeteerPage).clickOn("clip-commit");
+  async getElementCoordinates(testid) {
+    return page.$eval(`[data-testid='${testid}']`, elem => {
+      const { left, top } = elem.getBoundingClientRect();
+      return [left, top];
+    });
  }
 });
@@ -0,0 +1,35 @@
+import puppeteer from "puppeteer";
+import { DEBUG, DEV } from "./config";
+import { puppeteerUtils } from "./puppeteerUtils";
+import { cellxgeneActions } from "./cellxgeneActions";
+
+export async function setupTestBrowser(browserViewport) {
+  const browserParams = DEV
+    ? { headless: false, slowMo: 5 }
+    : DEBUG
+      ? { headless: false, slowMo: 100, devtools: true }
+      : {};
+  const browser = await puppeteer.launch(browserParams);
+  const page = await browser.pages().then(pages => pages[0]);
+  await page.setViewport(browserViewport);
+  if (DEV || DEBUG) {
+    page.on("console", async msg => {
+      // If there is a console.error but an error is not thrown, this will ensure the test fails
+      if (msg.type() === "error") {
+        const errorMsgText = await Promise.all(
+          // TODO can we do this without internal properties?
+          msg.args().map(arg => arg._remoteObject.description)
+        );
+        throw new Error(`Console error: ${errorMsgText}`);
+      }
+      console.log(`PAGE LOG: ${msg.text()}`);
+    });
+  }
+  page.on("pageerror", err => {
+    throw new Error(`Console error: ${err}`);
+  });
+  const utils = puppeteerUtils(page);
+  const cxgActions = cellxgeneActions(page, utils);
+  return [browser, page, utils, cxgActions];
+}
+
@@ -0,0 +1,96 @@
+import _ from "lodash";
+
+import calcCentroid from "../../src/util/centroid";
+
+import quantile from "../../src/util/quantile";
+import * as Universe from "../../src/util/stateManager/universe";
+import * as World from "../../src/util/stateManager/world";
+import * as REST from "./stateManager/sampleResponses";
+import { ControlsHelpers as CH } from "../../src/util/stateManager";
+
+describe("centroid", () => {
+  let world;
+  let categoricalSelection;
+  beforeAll(() => {
+    // Create world + universe
+    let universe = Universe.createUniverseFromResponse(
+      _.cloneDeep(REST.config),
+      _.cloneDeep(REST.schema)
+    );
+
+    universe = {
+      ...universe,
+      ...Universe.addObsAnnotations(
+        universe,
+        Universe.matrixFBSToDataframe(REST.annotationsObs)
+      ),
+      ...Universe.addVarAnnotations(
+        universe,
+        Universe.matrixFBSToDataframe(REST.annotationsVar)
+      ),
+      ...Universe.addObsLayout(
+        universe,
+        Universe.matrixFBSToDataframe(REST.layoutObs)
+      )
+    };
+    world = World.createWorldFromEntireUniverse(universe);
+
+    // Create categorical selection from world
+    categoricalSelection = CH.createCategoricalSelection(
+      world,
+      CH.selectableCategoryNames(world.schema, CH.maxCategoryItems(REST.config))
+    );
+  });
+
+  test("field4 (categorical obsAnnotation)", () => {
+    const centroidResult = calcCentroid(
+      world.obsAnnotations,
+      world.obsLayout,
+      "field4",
+      ["umap_0", "umap_1"],
+      categoricalSelection,
+      world.schema.annotations.obsByName
+    );
+
+    // Check to see that a centroid has been calculated for every categorical value
+    const keysAsArray = Array.from(centroidResult.keys());
+    expect(keysAsArray).toEqual(
+      expect.arrayContaining([83, true, "foo", 2.222222])
+    );
+
+    // This expected result assumes that all cells belong in all categorical values inside of sample response
+    const expectedResult = [
+      quantile([0.5], world.obsLayout.col("umap_0").asArray())[0],
+      quantile([0.5], world.obsLayout.col("umap_1").asArray())[0]
+    ];
+
+    centroidResult.forEach(coordinate => {
+      expect(coordinate).toEqual(expectedResult);
+    });
+  });
+
+  test("field3 (boolean obsAnnotation", () => {
+    const centroidResult = calcCentroid(
+      world.obsAnnotations,
+      world.obsLayout,
+      "field3",
+      ["umap_0", "umap_1"],
+      categoricalSelection,
+      world.schema.annotations.obsByName
+    );
+
+    // Check to see that a centroid has been calculated for every categorical value
+    const keysAsArray = Array.from(centroidResult.keys());
+    expect(keysAsArray).toEqual(expect.arrayContaining([false, true]));
+
+    // This expected result assumes that all cells belong in all categorical values inside of sample response
+    const expectedResult = [
+      quantile([0.5], world.obsLayout.col("umap_0").asArray())[0],
+      quantile([0.5], world.obsLayout.col("umap_1").asArray())[0]
+    ];
+
+    centroidResult.forEach(coordinate => {
+      expect(coordinate).toEqual(expectedResult);
+    });
+  });
+});
@@ -321,7 +321,10 @@ describe("dataframe factories", () => {
    test("KeyIndex", () => {
      const df = new Dataframe.Dataframe(
        [2, 2],
-        [["red", "blue"], [true, false]],
+        [
+          ["red", "blue"],
+          [true, false]
+        ],
        null,
        new Dataframe.KeyIndex(["colors", "bools"])
      );
@@ -341,7 +344,10 @@ describe("dataframe factories", () => {
    test("DenseInt32Index", () => {
      const df = new Dataframe.Dataframe(
        [2, 2],
-        [["red", "blue"], [true, false]],
+        [
+          ["red", "blue"],
+          [true, false]
+        ],
        null,
        new Dataframe.DenseInt32Index([74, 75])
      );
@@ -363,7 +369,10 @@ describe("dataframe factories", () => {
    test("DenseInt32Index promote", () => {
      const df = new Dataframe.Dataframe(
        [2, 2],
-        [["red", "blue"], [true, false]],
+        [
+          ["red", "blue"],
+          [true, false]
+        ],
        null,
        new Dataframe.DenseInt32Index([74, 75])
      );
@@ -385,7 +394,10 @@ describe("dataframe factories", () => {
    test("IdentityInt32Index with last", () => {
      const df = new Dataframe.Dataframe(
        [2, 2],
-        [["red", "blue"], [true, false]],
+        [
+          ["red", "blue"],
+          [true, false]
+        ],
        null,
        null
      );
@@ -407,7 +419,10 @@ describe("dataframe factories", () => {
    test("IdentityInt32Index promote", () => {
      const df = new Dataframe.Dataframe(
        [2, 2],
-        [["red", "blue"], [true, false]],
+        [
+          ["red", "blue"],
+          [true, false]
+        ],
        null,
        null
      );
@@ -461,7 +476,11 @@ describe("dataframe factories", () => {
      */
      const dfA = new Dataframe.Dataframe(
        [2, 3],
-        [["red", "blue"], [true, false], [1, 0]],
+        [
+          ["red", "blue"],
+          [true, false],
+          [1, 0]
+        ],
        null,
        new Dataframe.KeyIndex(["colors", "bools", "numbers"])
      );
@@ -481,6 +500,7 @@ describe("dataframe factories", () => {

    test("simple", () => {
      /* simple test that it works as expected in common case */
+      const dfEmpty = Dataframe.Dataframe.empty();
      const dfA = new Dataframe.Dataframe(
        [2, 1],
        [["red", "blue"]],
@@ -494,6 +514,22 @@ describe("dataframe factories", () => {
        new Dataframe.KeyIndex(["bools"])
      );

+      const dfLikeA = dfEmpty.withColsFrom(dfA);
+      expect(dfLikeA).toBeDefined();
+      expect(dfLikeA.dims).toEqual(dfA.dims);
+      expect(dfLikeA.colIndex.keys()).toEqual(dfA.colIndex.keys());
+      expect(dfLikeA.rowIndex).toEqual(dfA.rowIndex);
+      expect(dfLikeA.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(dfLikeA.icol(0).asArray()).toEqual(dfA.icol(0).asArray());
+
+      const dfAlsoLikeA = dfA.withColsFrom(dfEmpty);
+      expect(dfAlsoLikeA).toBeDefined();
+      expect(dfAlsoLikeA.dims).toEqual(dfA.dims);
+      expect(dfAlsoLikeA.colIndex.keys()).toEqual(dfA.colIndex.keys());
+      expect(dfAlsoLikeA.rowIndex).toEqual(dfA.rowIndex);
+      expect(dfAlsoLikeA.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(dfAlsoLikeA.icol(0).asArray()).toEqual(dfA.icol(0).asArray());
+
      const dfC = dfA.withColsFrom(dfB);
      expect(dfC).toBeDefined();
      expect(dfC.dims).toEqual([2, 2]);
@@ -503,13 +539,87 @@ describe("dataframe factories", () => {
      expect(dfC.col("colors").asArray()).toEqual(["red", "blue"]);
      expect(dfC.col("bools").asArray()).toEqual([true, false]);
    });
+
+    test("column picking", () => {
+      const dfEmpty = Dataframe.Dataframe.empty();
+      const dfA = new Dataframe.Dataframe(
+        [2, 1],
+        [["red", "blue"]],
+        null,
+        new Dataframe.KeyIndex(["colors"])
+      );
+      const dfB = new Dataframe.Dataframe(
+        [2, 3],
+        [
+          ["red", "blue"],
+          [true, false],
+          [1, 0]
+        ],
+        null,
+        new Dataframe.KeyIndex(["colors", "bools", "numbers"])
+      );
+
+      const dfX = dfEmpty.withColsFrom(dfB, ["colors", "bools"]);
+      expect(dfX).toBeDefined();
+      expect(dfX.dims).toEqual([2, 2]);
+      expect(dfX.colIndex.keys()).toEqual(["colors", "bools"]);
+      expect(dfX.rowIndex).toEqual(dfB.rowIndex);
+      expect(dfX.icol(0).asArray()).toEqual(dfB.icol(0).asArray());
+
+      const dfY = dfA.withColsFrom(dfB, ["numbers"]);
+      expect(dfY).toBeDefined();
+      expect(dfY.dims).toEqual([2, 2]);
+      expect(dfY.colIndex.keys()).toEqual(["colors", "numbers"]);
+      expect(dfY.rowIndex).toEqual(dfA.rowIndex);
+      expect(dfY.icol(0).asArray()).toEqual(dfA.icol(0).asArray());
+
+      const dfZ = dfA.withColsFrom(dfEmpty, []);
+      expect(dfZ).toBeDefined();
+      expect(dfZ.dims).toEqual(dfA.dims);
+      expect(dfZ.colIndex.keys()).toEqual(dfA.colIndex.keys());
+      expect(dfZ.rowIndex).toEqual(dfA.rowIndex);
+      expect(dfZ.icol(0).asArray()).toEqual(dfA.icol(0).asArray());
+
+      expect(() => dfA.withColsFrom(dfB, ["bools", "colors"])).toThrow();
+    });
+
+    test("column aliasing", () => {
+      const dfA = new Dataframe.Dataframe(
+        [2, 1],
+        [["red", "blue"]],
+        null,
+        new Dataframe.KeyIndex(["colors"])
+      );
+      const dfB = new Dataframe.Dataframe(
+        [2, 3],
+        [
+          ["red", "blue"],
+          [true, false],
+          [1, 0]
+        ],
+        null,
+        new Dataframe.KeyIndex(["colors", "bools", "numbers"])
+      );
+
+      const dfX = dfA.withColsFrom(dfB, { colors: "_colors", bools: "_bools" });
+      expect(dfX).toBeDefined();
+      expect(dfX.dims).toEqual([2, 3]);
+      expect(dfX.colIndex.keys()).toEqual(["colors", "_colors", "_bools"]);
+      expect(dfX.rowIndex).toEqual(dfA.rowIndex);
+      expect(dfX.icol(0).asArray()).toEqual(dfA.icol(0).asArray());
+      expect(dfX.col("_colors").asArray()).toBe(dfB.col("colors").asArray());
+    });
  });

  describe("dropCol", () => {
    test("KeyIndex", () => {
      const df = new Dataframe.Dataframe(
        [2, 3],
-        [["red", "blue"], [true, false], [1, 0]],
+        [
+          ["red", "blue"],
+          [true, false],
+          [1, 0]
+        ],
        null,
        new Dataframe.KeyIndex(["colors", "bools", "numbers"])
      );
@@ -528,7 +638,11 @@ describe("dataframe factories", () => {
    test("IdentityInt32Index drop first", () => {
      const df = new Dataframe.Dataframe(
        [2, 3],
-        [["red", "blue"], [true, false], [1, 0]],
+        [
+          ["red", "blue"],
+          [true, false],
+          [1, 0]
+        ],
        null,
        null
      );
@@ -548,7 +662,11 @@ describe("dataframe factories", () => {
    test("IdentityInt32Index drop last", () => {
      const df = new Dataframe.Dataframe(
        [2, 3],
-        [["red", "blue"], [true, false], [1, 0]],
+        [
+          ["red", "blue"],
+          [true, false],
+          [1, 0]
+        ],
        null,
        null
      );
@@ -568,7 +686,11 @@ describe("dataframe factories", () => {
    test("DenseInt32Index", () => {
      const df = new Dataframe.Dataframe(
        [2, 3],
-        [["red", "blue"], [true, false], [1, 0]],
+        [
+          ["red", "blue"],
+          [true, false],
+          [1, 0]
+        ],
        null,
        new Dataframe.DenseInt32Index([102, 101, 100])
      );
@@ -636,7 +758,10 @@ describe("dataframe factories", () => {
    test("renameCol", () => {
      const dfA = new Dataframe.Dataframe(
        [2, 2],
-        [[true, false], [1, 0]],
+        [
+          [true, false],
+          [1, 0]
+        ],
        null,
        new Dataframe.KeyIndex(["A", "B"])
      );
@@ -654,7 +779,10 @@ describe("dataframe col", () => {
  beforeEach(() => {
    df = new Dataframe.Dataframe(
      [2, 2],
-      [[true, false], [1, 0]],
+      [
+        [true, false],
+        [1, 0]
+      ],
      null,
      new Dataframe.KeyIndex(["A", "B"])
    );
@@ -0,0 +1,24 @@
+/*
+test controls helpers
+*/
+import { subsetAndResetGeneLists } from "../../../src/util/stateManager/controlsHelpers";
+import * as globals from "../../../src/globals";
+
+describe("controls helpers", () => {
+
+  test("subsetAndResetGeneLists", () => {
+    const geneList = [...Array(150).keys()].map(() =>
+      Math.random().toString(36).substring(2, 6) // random string of 4 characters
+    );
+    const state = {
+      userDefinedGenes: geneList.slice(0, 20),
+      diffexpGenes: geneList.slice(20),
+    };
+    const [newUserDefinedGenes, newDiffExpGenes] = subsetAndResetGeneLists(state);
+    expect(globals.maxUserDefinedGenes).toBeLessThan(globals.maxGenes);
+    expect(geneList.length).toBeGreaterThan(globals.maxGenes);
+    expect(newUserDefinedGenes).toHaveLength(globals.maxGenes);
+    expect(newUserDefinedGenes).toStrictEqual(geneList.slice(0, globals.maxGenes));
+    expect(newDiffExpGenes).toStrictEqual([]);
+  });
+});
@@ -30,15 +30,39 @@ describe("createUniverseFromResponse", () => {
    create a universe from sample data nad validate its shape & contents
    */
    const { nObs, nVar } = REST.schema.schema.dataframe;
-    const universe = Universe.createUniverseFromResponse(
+    let universe = Universe.createUniverseFromResponse(
      REST.config,
-      REST.schema,
-      REST.annotationsObs,
-      REST.annotationsVar,
-      REST.layoutObs
+      REST.schema
+    );
+    expect(universe).toBeDefined();
+    expect(universe).toMatchObject(
+      expect.objectContaining({
+        nObs,
+        nVar,
+        schema: REST.schema.schema,
+        obsAnnotations: expect.any(Dataframe.Dataframe),
+        varAnnotations: expect.any(Dataframe.Dataframe),
+        obsLayout: expect.any(Dataframe.Dataframe),
+        varData: expect.any(Dataframe.Dataframe)
+      })
    );

-    expect(universe).toBeDefined();
+    universe = {
+      ...universe,
+      ...Universe.addObsAnnotations(
+        universe,
+        Universe.matrixFBSToDataframe(REST.annotationsObs)
+      ),
+      ...Universe.addVarAnnotations(
+        universe,
+        Universe.matrixFBSToDataframe(REST.annotationsVar)
+      ),
+      ...Universe.addObsLayout(
+        universe,
+        Universe.matrixFBSToDataframe(REST.layoutObs)
+      )
+    };
+
    expect(universe).toMatchObject(
      expect.objectContaining({
        nObs,
@@ -17,13 +17,27 @@ the default REST test response.
 const defaultBigBang = () => {
  /* create unverse, world, crossfilter and dimensionMap */
  /* create universe */
-  const universe = Universe.createUniverseFromResponse(
+  let universe = Universe.createUniverseFromResponse(
    _.cloneDeep(REST.config),
-    _.cloneDeep(REST.schema),
-    _.cloneDeep(REST.annotationsObs),
-    _.cloneDeep(REST.annotationsVar),
-    _.cloneDeep(REST.layoutObs)
+    _.cloneDeep(REST.schema)
  );
+
+  universe = {
+    ...universe,
+    ...Universe.addObsAnnotations(
+      universe,
+      Universe.matrixFBSToDataframe(REST.annotationsObs)
+    ),
+    ...Universe.addVarAnnotations(
+      universe,
+      Universe.matrixFBSToDataframe(REST.annotationsVar)
+    ),
+    ...Universe.addObsLayout(
+      universe,
+      Universe.matrixFBSToDataframe(REST.layoutObs)
+    )
+  };
+
  /* create world */
  const world = World.createWorldFromEntireUniverse(universe);
  /* create crossfilter */
@@ -45,9 +59,9 @@ describe("createWorldFromEntireUniverse", () => {
    const universe = Universe.createUniverseFromResponse(
      _.cloneDeep(REST.config),
      _.cloneDeep(REST.schema),
-      _.cloneDeep(REST.annotationsObs),
-      _.cloneDeep(REST.annotationsVar),
-      _.cloneDeep(REST.layoutObs)
+      Universe.matrixFBSToDataframe(_.cloneDeep(REST.annotationsObs)),
+      Universe.matrixFBSToDataframe(_.cloneDeep(REST.annotationsVar)),
+      Universe.matrixFBSToDataframe(_.cloneDeep(REST.layoutObs))
    );
    expect(universe).toBeDefined();

@@ -183,3 +183,24 @@ describe("fillBySelection", () => {
    expect(arr).toEqual(truth);
  });
 });
+
+describe("wide bitarray", () => {
+  test.each([9, 30, 31, 32, 33, 54, 63, 64, 65, 127, 128, 129])(
+    "more than %d dimensions",
+    ndim => {
+      /* ensure we move across the uint boundary correctly */
+
+      const ba = new BitArray(defaultTestLength);
+      expect(ba).toBeDefined();
+
+      for (let i = 0; i < ndim; i += 1) {
+        expect(ba.allocDimension()).toEqual(i);
+      }
+      ba.freeDimension(0);
+      expect(ba.allocDimension()).toEqual(0);
+
+      ba.freeDimension(ndim - 1);
+      expect(ba.allocDimension()).toEqual(ndim - 1);
+    }
+  );
+});
@@ -88,6 +88,9 @@ module.exports = {
    new webpack.NoEmitOnErrorsPlugin(),
    new webpack.DefinePlugin({
      __REACT_DEVTOOLS_GLOBAL_HOOK__: "({ isDisabled: true })"
+    }),
+    new webpack.DefinePlugin({
+      "process.env.CXG_SERVER_PORT": JSON.stringify(process.env.CXG_SERVER_PORT),
    })
  ]
 };
@@ -1,24 +1,15 @@
 {
  "name": "cellxgene",
-  "version": "0.14.1",
+  "version": "0.15.0",
  "license": "MIT",
  "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
  "repository": "https://github.com/chanzuckerberg/cellxgene",
  "scripts": {
-    "backend-dev": "python3.6 -m venv cellxgene && source cellxgene/bin/activate && yes | pip uninstall cellxgene || true && pip install -e .. && cellxgene launch ",
-    "backend-dev-anno-ontology": "python3.6 -m venv cellxgene && source cellxgene/bin/activate && yes | pip uninstall cellxgene || true && pip install -e .. && cellxgene launch --experimental-annotations --experimental-annotations-ontology",
-    "backend-dev-anno": "python3.6 -m venv cellxgene && source cellxgene/bin/activate && yes | pip uninstall cellxgene || true && pip install -e .. && cellxgene launch --experimental-annotations",
-    "build": "npm run clean && webpack --config configuration/webpack/webpack.config.prod.js",
    "clean": "rimraf build",
+    "build": "npm run clean && webpack --config configuration/webpack/webpack.config.prod.js",
    "dev": "npm run clean && webpack --config configuration/webpack/webpack.config.dev.js",
-    "e2e": "node node_modules/jest/bin/jest.js --verbose false --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
    "fmt": "eslint --fix src",
-    "lint": "eslint src",
-    "smoke-test": "start-server-and-test start-server-for-test :5000 e2e",
-    "start": "node server/development.js",
-    "start-server-for-test": "cellxgene launch -p 5000 ../example-dataset/pbmc3k.h5ad",
-    "test": "node node_modules/jest/bin/jest.js",
-    "unit-test": "node node_modules/jest/bin/jest.js --testPathIgnorePatterns e2e"
+    "lint": "eslint src"
  },
  "engineStrict": true,
  "engines": {
@@ -34,80 +25,79 @@
    "eslint-scope": "3.7.1"
  },
  "dependencies": {
-    "@blueprintjs/core": "^3.22.3",
-    "@blueprintjs/icons": "^3.13.0",
-    "@blueprintjs/select": "^3.11.2",
+    "@blueprintjs/core": "^3.24.0",
+    "@blueprintjs/icons": "^3.14.0",
+    "@blueprintjs/select": "^3.12.0",
    "client": "0.0.1",
    "d3": "^4.10.0",
    "d3-scale-chromatic": "^1.5.0",
    "flatbuffers": "^1.11.0",
    "fuzzysort": "^1.1.4",
-    "gl-mat4": "^1.1.4",
-    "gl-matrix": "^3.1.0",
+    "gl-mat4": "^1.2.0",
+    "gl-matrix": "^3.2.1",
    "gl-vec3": "^1.1.3",
    "is-number": "^7.0.0",
    "lodash": "^4.17.15",
    "memoize-one": "^5.1.1",
-    "react": "^16.12.0",
-    "react-autocomplete": "^1.7.2",
-    "react-dom": "^16.12.0",
+    "react": "^16.13.0",
+    "react-dom": "^16.13.0",
    "react-flip-toolkit": "7.0.6",
    "react-helmet": "^5.2.1",
-    "react-icons": "^3.8.0",
-    "react-redux": "^7.1.3",
+    "react-icons": "^3.9.0",
+    "react-redux": "^7.2.0",
    "redux": "^4.0.5",
-    "redux-thunk": "^2.2.0",
-    "regl": "^1.3.13"
+    "redux-thunk": "^2.3.0",
+    "regl": "^1.4.0"
  },
  "devDependencies": {
-    "@babel/core": "^7.7.7",
-    "@babel/plugin-proposal-class-properties": "^7.7.4",
-    "@babel/plugin-proposal-decorators": "^7.7.4",
-    "@babel/plugin-proposal-export-namespace-from": "^7.7.4",
-    "@babel/plugin-proposal-function-bind": "^7.7.4",
-    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.7.4",
-    "@babel/plugin-proposal-optional-chaining": "^7.7.5",
-    "@babel/plugin-transform-react-constant-elements": "^7.7.4",
-    "@babel/plugin-transform-runtime": "^7.7.6",
-    "@babel/preset-env": "^7.7.7",
-    "@babel/preset-react": "^7.7.4",
-    "@babel/register": "^7.7.7",
-    "@babel/runtime": "^7.7.7",
-    "babel-eslint": "^10.0.3",
-    "babel-jest": "^24.9.0",
+    "@babel/core": "^7.8.7",
+    "@babel/plugin-proposal-class-properties": "^7.8.3",
+    "@babel/plugin-proposal-decorators": "^7.8.3",
+    "@babel/plugin-proposal-export-namespace-from": "^7.8.3",
+    "@babel/plugin-proposal-function-bind": "^7.8.3",
+    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.8.3",
+    "@babel/plugin-proposal-optional-chaining": "^7.8.3",
+    "@babel/plugin-transform-react-constant-elements": "^7.8.3",
+    "@babel/plugin-transform-runtime": "^7.8.3",
+    "@babel/preset-env": "^7.8.7",
+    "@babel/preset-react": "^7.8.3",
+    "@babel/register": "^7.8.6",
+    "@babel/runtime": "^7.8.7",
+    "babel-eslint": "^10.1.0",
+    "babel-jest": "^25.1.0",
    "babel-loader": "^8.0.6",
    "babel-preset-modern-browsers": "^14.1.1",
    "chalk": "^3.0.0",
+    "codecov": "^3.6.5",
    "connect-history-api-fallback": "^1.6.0",
    "copy-webpack-plugin": "^5.1.1",
-    "css-loader": "^3.4.1",
+    "css-loader": "^3.4.2",
    "eslint": "^6.8.0",
    "eslint-config-airbnb": "^18.0.1",
-    "eslint-config-prettier": "^6.9.0",
+    "eslint-config-prettier": "^6.10.0",
    "eslint-loader": "^3.0.3",
    "eslint-plugin-filenames": "^1.3.2",
-    "eslint-plugin-import": "^2.19.1",
-    "eslint-plugin-jest": "^23.3.0",
+    "eslint-plugin-import": "^2.20.1",
+    "eslint-plugin-jest": "^23.8.2",
    "eslint-plugin-jsx-a11y": "^6.2.3",
-    "eslint-plugin-react": "^7.17.0",
-    "eslint-plugin-react-hooks": "^2.3.0",
+    "eslint-plugin-react": "^7.19.0",
+    "eslint-plugin-react-hooks": "^2.5.0",
    "express": "^4.17.1",
-    "file-loader": "^5.0.2",
+    "file-loader": "^5.1.0",
    "html-webpack-inline-source-plugin": "0.0.10",
    "html-webpack-plugin": "^3.2.0",
-    "jest": "^24.9.0",
+    "jest": "^25.1.0",
    "jest-puppeteer": "^4.4.0",
-    "json-loader": "^0.5.4",
+    "json-loader": "^0.5.7",
    "mini-css-extract-plugin": "^0.9.0",
-    "puppeteer": "^2.0.0",
-    "rimraf": "^3.0.0",
-    "serve-favicon": "^2.3.0",
-    "start-server-and-test": "^1.10.6",
-    "style-loader": "^1.1.2",
-    "sw-precache-webpack-plugin": "^0.11.5",
+    "puppeteer": "^2.1.1",
+    "rimraf": "^3.0.2",
+    "serve-favicon": "^2.5.0",
+    "style-loader": "^1.1.3",
+    "sw-precache-webpack-plugin": "^1.0.0",
    "url-loader": "^3.0.0",
-    "webpack": "^4.41.5",
-    "webpack-cli": "^3.3.10",
+    "webpack": "^4.42.0",
+    "webpack-cli": "^3.3.11",
    "webpack-dev-middleware": "^3.7.2"
  },
  "jest": {
@@ -117,7 +107,9 @@
    "testURL": "http://localhost/",
    "setupFiles": [
      "./__tests__/setupMissingGlobals.js"
-    ]
+    ],
+    "coverageDirectory": "./coverage/",
+    "collectCoverage": true
  },
  "babel": {
    "env": {
@@ -11,7 +11,7 @@ var utils = require("./utils");

 process.env.NODE_ENV = "development";

-var PORT = process.env.PORT || 3000;
+const CLIENT_PORT = process.env.CXG_CLIENT_PORT;

 // Set up compiler
 var compiler = webpack(config);
@@ -22,7 +22,7 @@ compiler.plugin("invalid", () => {
 });

 compiler.plugin("done", stats => {
-  utils.formatStats(stats, PORT);
+  utils.formatStats(stats, CLIENT_PORT);
 });

 // Launch server
@@ -43,7 +43,7 @@ app.get("*", (req, res) => {
  res.sendFile(path.resolve("index.html"));
 });

-app.listen(PORT, err => {
+app.listen(CLIENT_PORT, err => {
  if (err) {
    console.log(err);
    return;
@@ -1,4 +1,3 @@
-// jshint esversion: 6
 import _ from "lodash";
 import * as globals from "../globals";
 import { Universe, MatrixFBS } from "../util/stateManager";
@@ -8,6 +7,90 @@ import {
  doBinaryRequest,
  dispatchNetworkErrorMessageToUser
 } from "../util/actionHelpers";
+import { requestReembed, reembedResetWorldToUniverse } from "./reembed";
+
+/*
+return promise to fetch the OBS annotations we need to load.  Omit anything
+we don't need.
+*/
+function obsAnnotationFetchAndLoad(dispatch, schema) {
+  const obsAnnotations = schema?.schema?.annotations?.obs ?? {};
+  const columns = obsAnnotations.columns ?? [];
+  const index = obsAnnotations.index ?? false;
+  return Promise.all(
+    columns
+      .filter(col => col.name !== index)
+      .map(col => {
+        const path = `annotations/obs?annotation-name=${encodeURIComponent(
+          col.name
+        )}`;
+        const url = `${globals.API.prefix}${globals.API.version}${path}`;
+        return doBinaryRequest(url);
+      })
+      .map(rqst => rqst.then(buffer => Universe.matrixFBSToDataframe(buffer)))
+      .map(resp =>
+        resp.then(df =>
+          dispatch({
+            type: "universe: column load success",
+            dim: "obsAnnotations",
+            dataframe: df
+          })
+        )
+      )
+  );
+}
+
+/*
+return promise fetching VAR annotations we need to load.  Only index is currently used.
+*/
+function varAnnotationFetchAndLoad(dispatch, schema) {
+  const varAnnotations = schema?.schema?.annotations?.var ?? {};
+  const index = varAnnotations.index ?? false;
+  const names = index ? [index] : [];
+  return Promise.all(
+    names
+      .map(name => {
+        const path = `annotations/var?annotation-name=${encodeURIComponent(
+          name
+        )}`;
+        const url = `${globals.API.prefix}${globals.API.version}${path}`;
+        return doBinaryRequest(url);
+      })
+      .map(rqst => rqst.then(buffer => Universe.matrixFBSToDataframe(buffer)))
+      .map(resp =>
+        resp.then(df =>
+          dispatch({
+            type: "universe: column load success",
+            dim: "varAnnotations",
+            dataframe: df
+          })
+        )
+      )
+  );
+}
+
+/*
+return promise fetching layout we need
+*/
+function layoutFetchAndLoad(dispatch) {
+  return Promise.all(
+    ["layout/obs"]
+      .map(path => {
+        const url = `${globals.API.prefix}${globals.API.version}${path}`;
+        return doBinaryRequest(url);
+      })
+      .map(rqst => rqst.then(buffer => Universe.matrixFBSToDataframe(buffer)))
+      .map(resp =>
+        resp.then(df =>
+          dispatch({
+            type: "universe: column load success",
+            dim: "obsLayout",
+            dataframe: df
+          })
+        )
+      )
+  );
+}

 /*
 Bootstrap application with the initial data loading.
@@ -31,34 +114,29 @@ const doInitialDataLoad = () =>
      /* set config defaults */
      const config = { ...globals.configDefaults, ...stepOneResults[0].config };
      const schema = stepOneResults[1];
-
-      /*
-      Step 2 - dataframes, all binary.  NOTE: uses results of step 1.
-      */
-      /* only load names for var annotations, if possible*/
-      const varIndexName = schema?.schema?.annotations?.var?.index;
-      const varAnnotationsQuery = varIndexName
-        ? `?annotation-name=${encodeURIComponent(varIndexName)}`
-        : "";
-      const varAnnotationsURL = `annotations/var${varAnnotationsQuery}`;
-      const requestBinary = ["annotations/obs", varAnnotationsURL, "layout/obs"]
-        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
-        .map(url => doBinaryRequest(url));
-      const stepTwoResults = await Promise.all(requestBinary);
-      const [obsAnno, varAnno, obsLayout] = [...stepTwoResults];
-
-      const universe = Universe.createUniverseFromResponse(
-        config,
-        schema,
-        obsAnno,
-        varAnno,
-        obsLayout
-      );
-
+      const universe = Universe.createUniverseFromResponse(config, schema);
+      dispatch({
+        type: "universe exists, but loading is still in progress",
+        universe
+      });
      dispatch({
        type: "configuration load complete",
        config
      });
+
+      /*
+      Step 2 - load the minimum stuff required to display.
+      */
+      await Promise.all([
+        layoutFetchAndLoad(dispatch),
+        varAnnotationFetchAndLoad(dispatch, schema)
+      ]);
+
+      /*
+      Step 3 - load everything else
+      */
+      await obsAnnotationFetchAndLoad(dispatch, schema);
+
      dispatch({
        type: "initial data load complete (universe exists)",
        universe
@@ -72,7 +150,7 @@ const doInitialDataLoad = () =>
 Set the view (world) to current selection.   Placeholder for an async action
 which also does re-layout.
 */
-const regraph = () => (dispatch, getState) => {
+const setWorldToSelection = () => (dispatch, getState) => {
  const { universe, world, crossfilter } = getState();
  dispatch({
    type: "set World to current selection",
@@ -90,7 +168,7 @@ const dispatchExpressionErrors = (dispatch, res) => {
 };

 /*
-Fetch expression vectors for each gene in genes.   This is NOT an action
+Fetch expression vectors for each gene in genes.  This is NOT an action
 function, but rather a helper to be called from an action helper that
 needs expression data.

@@ -308,37 +386,13 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
  }
 };

-const resetInterface = () => (dispatch, getState) => {
+const resetWorldToUniverse = () => (dispatch, getState) => {
  const { universe } = getState();
-
-  dispatch({
-    type: "user reset start"
-  });
-  dispatch({
-    type: "clear all user defined genes"
-  });
-  dispatch({
-    type: "clear differential expression"
-  });
-  dispatch({
-    type: "reset colorscale"
-  });
-  dispatch({
-    type: "reset centroid labels"
-  });
-  dispatch({
-    type: "clear scatterplot"
-  });
+  reembedResetWorldToUniverse(dispatch, getState);
  dispatch({
    type: "reset World to eq Universe",
    universe
  });
-  dispatch({
-    type: "increment graph render counter"
-  });
-  dispatch({
-    type: "user reset end"
-  });
 };

 const saveObsAnnotations = () => async (dispatch, getState) => {
@@ -395,11 +449,12 @@ const saveObsAnnotations = () => async (dispatch, getState) => {
 };

 export default {
-  regraph,
-  resetInterface,
-  requestSingleGeneExpressionCountsForColoringPOST,
-  requestDifferentialExpression,
-  requestUserDefinedGene,
  doInitialDataLoad,
-  saveObsAnnotations
+  requestDifferentialExpression,
+  requestSingleGeneExpressionCountsForColoringPOST,
+  requestUserDefinedGene,
+  requestReembed,
+  resetWorldToUniverse,
+  saveObsAnnotations,
+  setWorldToSelection
 };
@@ -0,0 +1,113 @@
+import { API } from "../globals";
+import { Universe } from "../util/stateManager";
+import {
+  postNetworkErrorToast,
+  postAsyncSuccessToast,
+  postAsyncFailureToast
+} from "../components/framework/toasters";
+
+function abortableFetch(request, opts, timeout = 0) {
+  const controller = new AbortController();
+  const { signal } = controller;
+
+  return {
+    abort: () => controller.abort(),
+    isAborted: () => signal.aborted,
+    ready: () => {
+      if (timeout) {
+        setTimeout(() => controller.abort(), timeout);
+      }
+      return fetch(request, { ...opts, signal });
+    }
+  };
+}
+
+async function doReembedFetch(dispatch, getState) {
+  const state = getState();
+  let cells = state.world.obsAnnotations.rowIndex.keys();
+
+  // These lines ensure that we convert any TypedArray to an Array.
+  // This is necessary because JSON.stringify() does some very strange
+  // things with TypedArrays (they are marshalled to JSON objects, rather
+  // than being marshalled as a JSON array).
+  cells = Array.isArray(cells) ? cells : Array.from(cells);
+
+  const af = abortableFetch(
+    `${API.prefix}${API.version}layout/obs`,
+    {
+      method: "PUT",
+      headers: new Headers({
+        Accept: "application/octet-stream",
+        "Content-Type": "application/json"
+      }),
+      body: JSON.stringify({
+        method: "umap",
+        filter: { obs: { index: cells } }
+      }),
+      credentials: "include"
+    },
+    60000 // 1 minute timeout
+  );
+  dispatch({
+    type: "reembed: request start",
+    abortableFetch: af
+  });
+  const res = await af.ready();
+
+  if (
+    res.ok &&
+    res.headers.get("Content-Type").includes("application/octet-stream")
+  ) {
+    return res;
+  }
+
+  // else an error
+  let msg = `Unexpected HTTP response ${res.status}, ${res.statusText}`;
+  const body = await res.text();
+  if (body && body.length > 0) {
+    msg = `${msg} -- ${body}`;
+  }
+  postNetworkErrorToast(msg);
+  throw new Error(msg);
+}
+
+/*
+functions below are dispatch-able
+*/
+export function requestReembed() {
+  return async (dispatch, getState) => {
+    try {
+      const res = await doReembedFetch(dispatch, getState);
+      const schema = JSON.parse(res.headers.get("CxG-Schema"));
+      const buffer = await res.arrayBuffer();
+      const df = Universe.matrixFBSToDataframe(buffer);
+      dispatch({
+        type: "reembed: request completed"
+      });
+      dispatch({
+        type: "reembed: add reembedding",
+        embedding: df,
+        schema
+      });
+      postAsyncSuccessToast("Re-embedding has completed.");
+    } catch (error) {
+      dispatch({
+        type: "reembed: request aborted"
+      });
+      if (error.name === "AbortError") {
+        postAsyncFailureToast("Re-embedding calculation was aborted.");
+      } else {
+        postNetworkErrorToast(`Re-embedding: ${error.message}`);
+      }
+      console.log("Reembed exception:", error, error.name, error.message);
+    }
+  };
+}
+
+export function reembedResetWorldToUniverse(dispatch, getState) {
+  const { reembedController } = getState();
+  if (reembedController.pendingFetch) reembedController.pendingFetch.abort();
+  dispatch({
+    type: "reembed: clear all reembeddings"
+  });
+}
@@ -74,7 +74,7 @@ class FilenameDialog extends React.Component {
            color: Colors.ORANGE3
          }}
        >
-          {"Name cannot be blank"}
+          Name cannot be blank
        </span>
      );
    } else if (err === "characters") {
@@ -87,7 +87,7 @@ class FilenameDialog extends React.Component {
            color: Colors.ORANGE3
          }}
        >
-          {"Only alphanumeric and underscore allowed"}
+          Only alphanumeric and underscore allowed
        </span>
      );
    }
@@ -11,7 +11,8 @@ import FilenameDialog from "./filenameDialog";
  saveInProgress: state.autosave?.saveInProgress ?? false,
  lastSavedObsAnnotations: state.autosave?.lastSavedObsAnnotations,
  error: state.autosave?.error,
-  writableCategoriesEnabled: state.config?.parameters?.["annotations"] ?? false
+  writableCategoriesEnabled: state.config?.parameters?.["annotations"] ?? false,
+  initialDataLoadComplete: state.autosave?.initialDataLoadComplete
 }))
 class Autosave extends React.Component {
  constructor(props) {
@@ -63,10 +64,17 @@ class Autosave extends React.Component {
  }

  render() {
-    const { writableCategoriesEnabled } = this.props;
+    const { writableCategoriesEnabled, saveInProgress, initialDataLoadComplete } = this.props;
    return writableCategoriesEnabled ? (
      <div
        id="autosave"
+        data-testclass={
+          !initialDataLoadComplete
+            ? "autosave-init"
+            : (this.needToSave() || saveInProgress)
+              ? "autosave-incomplete"
+              : "autosave-complete"
+        }
        style={{
          position: "fixed",
          display: "inherit",
@@ -351,8 +351,8 @@ class HistogramBrush extends React.PureComponent {
    const brushX = d3
      .brushX()
      .extent([
-          [x.range()[0], y.range()[1]],
-          [x.range()[1], this.marginTop + this.height + this.marginBottom]
+        [x.range()[0], y.range()[1]],
+        [x.range()[1], this.marginTop + this.height + this.marginBottom]
      ])
      /*
      emit start so that the Undoable history can save an undo point
@@ -6,11 +6,9 @@ import { Button, Tooltip, Dialog, Classes, Colors } from "@blueprintjs/core";
  colorAccessor: state.colors.colorAccessor,
  categoricalSelection: state.categoricalSelection,
  annotations: state.annotations,
-  universe: state.universe,
-  ontology: state.ontology,
-  ontologyLoading: state.ontology?.loading
+  universe: state.universe
 }))
-class AnnoDialog extends React.Component {
+class AnnoDialog extends React.PureComponent {
  constructor(props) {
    super(props);
    this.state = {};
@@ -26,13 +24,13 @@ class AnnoDialog extends React.Component {
      errorMessage,
      validationError,
      annoSelect,
-      ontologySelect,
      annoInput,
      handleCancel,
      handleSubmit,
      primaryButtonText,
      secondaryButtonText,
-      handleSecondaryButtonSubmit
+      handleSecondaryButtonSubmit,
+      primaryButtonProps
    } = this.props;

    return (
@@ -57,7 +55,6 @@ class AnnoDialog extends React.Component {
              </p>
            </div>
            {annoSelect || null}
-            {ontologySelect || null}
          </div>
          <div className={Classes.DIALOG_FOOTER}>
            <div className={Classes.DIALOG_FOOTER_ACTIONS}>
@@ -69,12 +66,13 @@ class AnnoDialog extends React.Component {
                  onClick={handleSecondaryButtonSubmit}
                  disabled={!text || validationError}
                  intent="none"
-                  type="submit"
+                  type="button"
                >
                  {secondaryButtonText}
                </Button>
              ) : null}
              <Button
+                {...primaryButtonProps} // eslint-disable-line react/jsx-props-no-spreading
                onClick={handleSubmit}
                disabled={!text || validationError}
                intent="primary"
@@ -1,9 +1,8 @@
 import React from "react";
 import { connect } from "react-redux";
 import AnnoDialog from "./annoDialog";
-import AnnoInputs from "./annoInputs";
-
-import { labelErrorMessage, isLabelErroneous } from "./labelUtil";
+import LabelInput from "./labelInput";
+import { labelPrompt, isLabelErroneous } from "./labelUtil";

@connect(state => ({
  colorAccessor: state.colors.colorAccessor,
@@ -11,10 +10,9 @@ import { labelErrorMessage, isLabelErroneous } from "./labelUtil";
  annotations: state.annotations,
  universe: state.universe,
  ontology: state.ontology,
-  ontologyLoading: state.ontology?.loading,
-  ontologyEnabled: state.ontology?.enabled
+  crossfilter: state.crossfilter
 }))
-class Category extends React.Component {
+class Category extends React.PureComponent {
  constructor(props) {
    super(props);
    this.state = {
@@ -22,53 +20,43 @@ class Category extends React.Component {
    };
  }

-  disableAddNewLabelMode = () => {
+  disableAddNewLabelMode = e => {
    const { dispatch } = this.props;
-    dispatch({
-      type: "annotation: disable add new label mode"
-    });
    this.setState({
      newLabelText: ""
    });
+    dispatch({
+      type: "annotation: disable add new label mode"
+    });
+    if (e) e.preventDefault();
  };

-  handleAddNewLabelToCategory = () => {
+  handleAddNewLabelToCategory = e => {
    const { dispatch, metadataField } = this.props;
    const { newLabelText } = this.state;

+    this.disableAddNewLabelMode();
    dispatch({
      type: "annotation: add new label to category",
      metadataField,
      newLabelText,
      assignSelectedCells: false
    });
-    this.setState({ newLabelText: "" });
+    e.preventDefault();
  };

-  addLabelAndAssignCells = () => {
+  addLabelAndAssignCells = e => {
    const { dispatch, metadataField } = this.props;
    const { newLabelText } = this.state;

+    this.disableAddNewLabelMode();
    dispatch({
      type: "annotation: add new label to category",
      metadataField,
      newLabelText,
      assignSelectedCells: true
    });
-
-    this.setState({ newLabelText: "" });
-  };
-
-  handleCreateArbitraryLabel = newLabelTextNotInOntology => {
-    const { dispatch, metadataField } = this.props;
-
-    dispatch({
-      type: "annotation: add new label to category",
-      metadataField,
-      newLabelText: newLabelTextNotInOntology,
-      assignSelectedCells: false
-    });
-    this.setState({ newLabelText: "" });
+    e.preventDefault();
  };

  labelNameError = name => {
@@ -76,23 +64,18 @@ class Category extends React.Component {
    return isLabelErroneous(name, metadataField, ontology, universe.schema);
  };

-  labelNameErrorMessage = name => {
-    const { metadataField, ontology, universe } = this.props;
-    return labelErrorMessage(name, metadataField, ontology, universe.schema);
+  instruction = label => {
+    return labelPrompt(this.labelNameError(label), "New, unique label", ":");
  };

-  /* leaky to have both of these in multiple components */
-  handleChoice = e => {
-    this.setState({ newLabelText: e.target });
-  };
-
-  handleTextChange = text => {
-    this.setState({ newLabelText: text });
+  handleChangeOrSelect = label => {
+    this.setState({ newLabelText: label });
  };

  render() {
    const { newLabelText } = this.state;
-    const { metadataField, annotations, ontologyEnabled } = this.props;
+    const { metadataField, annotations, ontology, crossfilter } = this.props;
+    const ontologyEnabled = ontology?.enabled ?? false;

    return (
      <>
@@ -101,27 +84,31 @@ class Category extends React.Component {
            annotations.isAddingNewLabel &&
            annotations.categoryAddingNewLabel === metadataField
          }
+          inputProps={{ "data-testid": `${metadataField}:create-label-dialog` }}
+          primaryButtonProps={{
+            "data-testid": `${metadataField}:submit-label`
+          }}
          title="Add new label to category"
-          instruction="New, unique label name:"
+          instruction={this.instruction(newLabelText)}
          cancelTooltipContent="Close this dialog without adding a label."
          primaryButtonText="Add label"
-          secondaryButtonText="Add label and assign currently selected cells"
+          secondaryButtonText={`Add label and assign ${crossfilter.countSelected()} currently selected cells`}
          handleSecondaryButtonSubmit={this.addLabelAndAssignCells}
          text={newLabelText}
          validationError={this.labelNameError(newLabelText)}
-          errorMessage={this.labelNameErrorMessage(newLabelText)}
          handleSubmit={this.handleAddNewLabelToCategory}
          handleCancel={this.disableAddNewLabelMode}
          annoInput={
-            <AnnoInputs
-              useSuggest={ontologyEnabled}
-              text={newLabelText}
-              handleCreateArbitraryLabel={this.handleCreateArbitraryLabel}
-              handleItemChange={this.handleSuggestActiveItemChange}
-              handleChoice={this.handleChoice}
-              handleTextChange={this.handleTextChange}
-              isTextInvalid={this.labelNameError}
-              isTextInvalidErrorMessage={this.labelNameErrorMessage}
+            <LabelInput
+              labelSuggestions={ontologyEnabled ? ontology.terms : null}
+              onChange={this.handleChangeOrSelect}
+              onSelect={this.handleChangeOrSelect}
+              inputProps={{
+                "data-testid": `${metadataField}:new-label-name`,
+                leftIcon: "tag",
+                intent: "none",
+                autoFocus: true
+              }}
            />
          }
        />
@@ -1,131 +0,0 @@
-import React from "react";
-import { connect } from "react-redux";
-import AnnoDialog from "./annoDialog";
-import OntologySelect from "./ontologySelect";
-
-import { labelErrorMessage, isLabelErroneous } from "./labelUtil";
-
-@connect(state => ({
-  colorAccessor: state.colors.colorAccessor,
-  categoricalSelection: state.categoricalSelection,
-  annotations: state.annotations,
-  universe: state.universe,
-  ontology: state.ontology,
-  ontologyLoading: state.ontology?.loading,
-  ontologyEnabled: state.ontology?.enabled
-}))
-class Category extends React.Component {
-  constructor(props) {
-    super(props);
-    this.state = {
-      newLabelText: ""
-    };
-  }
-
-  disableAddNewLabelFromOntologyMode = () => {
-    const { dispatch } = this.props;
-    dispatch({
-      type: "annotation: disable add new ontology label mode"
-    });
-    this.setState({
-      newLabelText: ""
-    });
-  };
-
-  handleAddNewLabelToCategory = () => {
-    const { dispatch, metadataField } = this.props;
-    const { newLabelText } = this.state;
-
-    dispatch({
-      type: "annotation: add new label to category",
-      metadataField,
-      newLabelText,
-      assignSelectedCells: false
-    });
-    this.setState({ newLabelText: "" });
-  };
-
-  addLabelAndAssignCells = () => {
-    const { dispatch, metadataField } = this.props;
-    const { newLabelText } = this.state;
-
-    dispatch({
-      type: "annotation: add new label to category",
-      metadataField,
-      newLabelText,
-      assignSelectedCells: true
-    });
-
-    this.setState({ newLabelText: "" });
-  };
-
-  handleCreateArbitraryLabel = newLabelTextNotInOntology => {
-    const { dispatch, metadataField } = this.props;
-
-    dispatch({
-      type: "annotation: add new label to category",
-      metadataField,
-      newLabelText: newLabelTextNotInOntology,
-      assignSelectedCells: false
-    });
-    this.setState({ newLabelText: "" });
-  };
-
-  labelNameError = name => {
-    const { metadataField, ontology, universe } = this.props;
-    return isLabelErroneous(name, metadataField, ontology, universe.schema);
-  };
-
-  labelNameErrorMessage = name => {
-    const { metadataField, ontology, universe } = this.props;
-    return labelErrorMessage(name, metadataField, ontology, universe.schema);
-  };
-
-  /* leaky to have both of these in multiple components */
-  handleChoice = e => {
-    this.setState({ newLabelText: e.target });
-  };
-
-  handleTextChange = text => {
-    this.setState({ newLabelText: text });
-  };
-
-  render() {
-    const { newLabelText } = this.state;
-    const { metadataField, annotations, ontology } = this.props;
-
-    return (
-      <>
-        <AnnoDialog
-          isActive={
-            annotations.isAddingNewLabelFromOntology &&
-            annotations.categoryAddingNewLabelFromOntology === metadataField
-          }
-          title="Add new label to category from existing ontology terms"
-          instruction="Choose an ontology term to use as a label name:"
-          cancelTooltipContent="Close this dialog without adding a label."
-          primaryButtonText="Add label"
-          secondaryButtonText="Add label and assign currently selected cells"
-          handleSecondaryButtonSubmit={this.addLabelAndAssignCells}
-          text={newLabelText}
-          validationError={this.labelNameError(newLabelText)}
-          errorMessage={this.labelNameErrorMessage(newLabelText)}
-          handleSubmit={this.handleAddNewLabelToCategory}
-          handleCancel={this.disableAddNewLabelFromOntologyMode}
-          ontologySelect={
-            <OntologySelect
-              handleChooseOntologyTermFromDropdown={term => {
-                this.handleChoice(term);
-              }}
-              categoryToDuplicate={newLabelText}
-              ontology={ontology}
-            />
-          }
-          annoInput={null}
-        />
-      </>
-    );
-  }
-}
-
-export default Category;
@@ -1,9 +1,9 @@
 import React from "react";
 import _ from "lodash";
 import { connect } from "react-redux";
-import { Colors } from "@blueprintjs/core";
 import AnnoDialog from "./annoDialog";
-import AnnoInputs from "./annoInputs";
+import LabelInput from "./labelInput";
+import { labelPrompt } from "./labelUtil";

 import { AnnotationsHelpers } from "../../util/stateManager";

@@ -11,11 +11,9 @@ import { AnnotationsHelpers } from "../../util/stateManager";
  categoricalSelection: state.categoricalSelection,
  annotations: state.annotations,
  universe: state.universe,
-  ontology: state.ontology,
-  ontologyLoading: state.ontology?.loading,
-  ontologyEnabled: state.ontology?.enabled
+  ontology: state.ontology
 }))
-class AnnoDialogEditCategoryName extends React.Component {
+class AnnoDialogEditCategoryName extends React.PureComponent {
  constructor(props) {
    super(props);
    this.state = {
@@ -23,20 +21,21 @@ class AnnoDialogEditCategoryName extends React.Component {
    };
  }

-  handleCategoryEditTextChange = txt => {
+  handleChangeOrSelect = name => {
    this.setState({
-      newCategoryText: txt
+      newCategoryText: name
    });
  };

  disableEditCategoryMode = () => {
-    const { dispatch } = this.props;
+    const { dispatch, metadataField } = this.props;
    dispatch({
      type: "annotation: disable category edit mode"
    });
+    this.setState({ newCategoryText: metadataField });
  };

-  handleEditCategory = () => {
+  handleEditCategory = e => {
    const { dispatch, metadataField, categoricalSelection } = this.props;
    const { newCategoryText } = this.state;

@@ -50,58 +49,29 @@ class AnnoDialogEditCategoryName extends React.Component {
      return;
    }

+    this.disableEditCategoryMode();
    dispatch({
      type: "annotation: category edited",
      metadataField,
      newCategoryText,
      data: newCategoryText
    });
+    e.preventDefault();
  };

-  categoryNameErrorMessage = () => {
-    const err = this.editedCategoryNameError();
-    if (err === false) return null;
-
-    const errorMessageMap = {
-      /* map error code to human readable error message */
-      "empty-string": "Blank names not allowed",
-      duplicate: "Category name must be unique",
-      "trim-spaces": "Leading and trailing spaces not allowed",
-      "illegal-characters":
-        "Only alphanumeric and special characters (-_.) allowed",
-      "multi-space-run": "Multiple consecutive spaces not allowed"
-    };
-    const errorMessage = errorMessageMap[err] ?? "error";
-    return (
-      <span
-        style={{
-          display: "block",
-          fontStyle: "italic",
-          fontSize: 12,
-          marginTop: 5,
-          color: Colors.ORANGE3
-        }}
-      >
-        {errorMessage}
-      </span>
-    );
-  };
-
-  editedCategoryNameError = () => {
+  editedCategoryNameError = name => {
    const { metadataField, categoricalSelection } = this.props;
-    const { newCategoryText } = this.state;

    /* check for syntax errors in category name */
-    const error = AnnotationsHelpers.annotationNameIsErroneous(newCategoryText);
+    const error = AnnotationsHelpers.annotationNameIsErroneous(name);
    if (error) {
      return error;
    }

    /* check for duplicative categories */
    const allCategoryNames = _.keys(categoricalSelection);
-    const categoryNameAlreadyExists =
-      allCategoryNames.indexOf(newCategoryText) > -1;
-    const sameName = newCategoryText === metadataField;
+    const categoryNameAlreadyExists = allCategoryNames.indexOf(name) > -1;
+    const sameName = name === metadataField;
    if (categoryNameAlreadyExists && !sameName) {
      return "duplicate";
    }
@@ -110,9 +80,18 @@ class AnnoDialogEditCategoryName extends React.Component {
    return false;
  };

+  instruction = name => {
+    return labelPrompt(
+      this.editedCategoryNameError(name),
+      "New, unique category name",
+      ":"
+    );
+  };
+
  render() {
    const { newCategoryText } = this.state;
-    const { metadataField, annotations } = this.props;
+    const { metadataField, annotations, ontology } = this.props;
+    const ontologyEnabled = ontology?.enabled ?? false;

    return (
      <>
@@ -121,20 +100,33 @@ class AnnoDialogEditCategoryName extends React.Component {
            annotations.isEditingCategoryName &&
            annotations.categoryBeingEdited === metadataField
          }
+          inputProps={{
+            "data-testid": `${metadataField}:edit-category-name-dialog`
+          }}
+          primaryButtonProps={{
+            "data-testid": `${metadataField}:submit-category-edit`
+          }}
          title="Edit category name"
-          instruction="New, unique category name:"
+          instruction={this.instruction(newCategoryText)}
          cancelTooltipContent="Close this dialog without editing this category."
          primaryButtonText="Edit category name"
          text={newCategoryText}
          validationError={this.editedCategoryNameError(newCategoryText)}
-          errorMessage={this.categoryNameErrorMessage(newCategoryText)}
          handleSubmit={this.handleEditCategory}
          handleCancel={this.disableEditCategoryMode}
          annoInput={
-            <AnnoInputs
-              useSuggest={false}
-              text={newCategoryText}
-              handleTextChange={this.handleCategoryEditTextChange}
+            <LabelInput
+              label={newCategoryText}
+              labelSuggestions={ontologyEnabled ? ontology.terms : null}
+              onChange={this.handleChangeOrSelect}
+              onSelect={this.handleChangeOrSelect}
+              inputProps={{
+                "data-testid": `${metadataField}:edit-category-name-text`,
+                leftIcon: "tag",
+                intent: "none",
+                autoFocus: true
+              }}
+              newLabelMessage="New category"
            />
          }
        />
@@ -1,43 +0,0 @@
-import React from "react";
-import { connect } from "react-redux";
-import { InputGroup } from "@blueprintjs/core";
-
-const VanillaInput = props => {
-  const { text, handleTextChange } = props;
-  return (
-    <InputGroup
-      autoFocus
-      value={text}
-      intent="none"
-      onChange={e => handleTextChange?.(e.target.value)}
-      leftIcon="tag"
-    />
-  );
-};
-
-@connect(state => ({
-  colorAccessor: state.colors.colorAccessor,
-  categoricalSelection: state.categoricalSelection,
-  annotations: state.annotations,
-  universe: state.universe,
-  world: state.world,
-  ontology: state.ontology?.terms,
-  ontologyLoading: state.ontology?.loading
-}))
-class AnnoInputs extends React.Component {
-  constructor(props) {
-    super(props);
-    this.state = {};
-  }
-
-  render() {
-    const { handleTextChange, text } = this.props;
-    return (
-      <div>
-        <VanillaInput text={text} handleTextChange={handleTextChange} />
-      </div>
-    );
-  }
-}
-
-export default AnnoInputs;
@@ -10,10 +10,9 @@ import {
 } from "@blueprintjs/core";

@connect(state => ({
-  annotations: state.annotations,
-  ontologyEnabled: state.ontology?.enabled
+  annotations: state.annotations
 }))
-class AnnoMenuCategory extends React.Component {
+class AnnoMenuCategory extends React.PureComponent {
  constructor(props) {
    super(props);
    this.state = {};
@@ -27,14 +26,6 @@ class AnnoMenuCategory extends React.Component {
    });
  };

-  activateAddNewOntologyLabelMode = () => {
-    const { dispatch, metadataField } = this.props;
-    dispatch({
-      type: "annotation: activate add new ontology label mode",
-      data: metadataField
-    });
-  };
-
  activateEditCategoryMode = () => {
    const { dispatch, metadataField } = this.props;

@@ -57,9 +48,7 @@ class AnnoMenuCategory extends React.Component {
      metadataField,
      annotations,
      isUserAnno,
-      ontologyEnabled,
      createText,
-      createFromOntologyText,
      editText,
      deleteText
    } = this.props;
@@ -76,24 +65,15 @@ class AnnoMenuCategory extends React.Component {
                <MenuItem
                  icon="tag"
                  data-testclass="handleAddNewLabelToCategory"
-                  data-testid={`handleAddNewLabelToCategory-${metadataField}`}
+                  data-testid={`${metadataField}:add-new-label-to-category`}
                  onClick={this.activateAddNewLabelMode}
                  text={createText}
                />
-                {ontologyEnabled ? (
-                  <MenuItem
-                    icon="book"
-                    data-testclass="activateAddNewOntologyLabelMode"
-                    data-testid={`activateAddNewOntologyLabelMode-${metadataField}`}
-                    onClick={this.activateAddNewOntologyLabelMode}
-                    text={createFromOntologyText}
-                  />
-                ) : null}
                <MenuItem
                  icon="edit"
                  disabled={annotations.isEditingCategoryName}
                  data-testclass="activateEditCategoryMode"
-                  data-testid={`activateEditCategoryMode-${metadataField}`}
+                  data-testid={`${metadataField}:edit-category-mode`}
                  onClick={this.activateEditCategoryMode}
                  text={editText}
                />
@@ -101,7 +81,7 @@ class AnnoMenuCategory extends React.Component {
                  icon="delete"
                  intent="danger"
                  data-testclass="handleDeleteCategory"
-                  data-testid={`handleDeleteCategory-${metadataField}`}
+                  data-testid={`${metadataField}:delete-category`}
                  onClick={this.handleDeleteCategory}
                  text={deleteText}
                />
@@ -111,7 +91,7 @@ class AnnoMenuCategory extends React.Component {
            <Button
              style={{ marginLeft: 0 }}
              data-testclass="seeActions"
-              data-testid={`seeActions-${metadataField}`}
+              data-testid={`${metadataField}:see-actions`}
              icon="more"
              minimal
            />
@@ -7,11 +7,9 @@ import { Select } from "@blueprintjs/select";
  colorAccessor: state.colors.colorAccessor,
  categoricalSelection: state.categoricalSelection,
  annotations: state.annotations,
-  universe: state.universe,
-  ontology: state.ontology,
-  ontologyLoading: state.ontology?.loading
+  universe: state.universe
 }))
-class DuplicateCategorySelect extends React.Component {
+class DuplicateCategorySelect extends React.PureComponent {
  constructor(props) {
    super(props);
    this.state = {};
@@ -4,15 +4,17 @@ import { Button } from "@blueprintjs/core";
 import { connect } from "react-redux";
 import * as globals from "../../globals";
 import Category from "./category";
-import { AnnotationsHelpers } from "../../util/stateManager";
+import { AnnotationsHelpers, ControlsHelpers } from "../../util/stateManager";
 import AnnoDialog from "./annoDialog";
-import AnnoInputs from "./annoInputs";
 import AnnoSelect from "./annoSelect";
+import LabelInput from "./labelInput";
+import { labelPrompt } from "./labelUtil";

@connect(state => ({
-  categoricalSelection: state.categoricalSelection,
  writableCategoriesEnabled: state.config?.parameters?.["annotations"] ?? false,
-  schema: state.world?.schema
+  schema: state.world?.schema,
+  config: state.config,
+  ontology: state.ontology
 }))
 class Categories extends React.Component {
  constructor(props) {
@@ -20,11 +22,12 @@ class Categories extends React.Component {
    this.state = {
      createAnnoModeActive: false,
      newCategoryText: "",
-      categoryToDuplicate: null
+      categoryToDuplicate: null,
+      expandedCats: new Set()
    };
  }

-  handleCreateUserAnno = () => {
+  handleCreateUserAnno = e => {
    const { dispatch } = this.props;
    const { newCategoryText, categoryToDuplicate } = this.state;
    dispatch({
@@ -37,6 +40,7 @@ class Categories extends React.Component {
      categoryToDuplicate: null,
      newCategoryText: ""
    });
+    e.preventDefault();
  };

  handleEnableAnnoMode = () => {
@@ -86,49 +90,49 @@ class Categories extends React.Component {
    return false;
  };

-  categoryNameErrorMessage = name => {
-    const err = this.categoryNameError(name);
-    if (err === false) return null;
-
-    const errorMessageMap = {
-      /* map error code to human readable error message */
-      "empty-string": "Blank names not allowed",
-      duplicate: "Name must be unique",
-      "trim-spaces": "Leading and trailing spaces not allowed",
-      "illegal-characters":
-        "Only alphanumeric and special characters (-_.) allowed",
-      "multi-space-run": "Multiple consecutive spaces not allowed"
-    };
-    const errorMessage = errorMessageMap[err] ?? "error";
-    return <span>{errorMessage}</span>;
+  handleChange = name => {
+    this.setState({ newCategoryText: name });
  };

-  handleNewCategoryText = txt => {
-    this.setState({ newCategoryText: txt });
+  handleSelect = name => {
+    this.setState({ newCategoryText: name });
  };

-  handleChoice = e => {
-    /* Blueprint Suggest format */
-    this.setState({ newCategoryText: e.target });
+  instruction = name => {
+    return labelPrompt(
+      this.categoryNameError(name),
+      "New, unique category name",
+      ":"
+    );
  };

-  handleSuggestActiveItemChange = () => {};
+  onExpansionChange = catName => {
+    const {expandedCats} = this.state;
+    if (expandedCats.has(catName)) {
+      const _expandedCats = new Set(expandedCats);
+      _expandedCats.delete(catName)
+      this.setState({expandedCats: _expandedCats}) 
+    } else {
+      const _expandedCats = new Set(expandedCats);
+      _expandedCats.add(catName)
+      this.setState({expandedCats: _expandedCats})
+    }
+  }

  render() {
    const {
      createAnnoModeActive,
      categoryToDuplicate,
-      newCategoryText
+      newCategoryText,
+      expandedCats      
    } = this.state;
-    const {
-      categoricalSelection,
-      writableCategoriesEnabled,
-      schema
-    } = this.props;
-    if (!categoricalSelection) return null;
-
+    const { writableCategoriesEnabled, schema, config, ontology } = this.props;
+    const ontologyEnabled = ontology?.enabled ?? false;
    /* all names, sorted in display order.  Will be rendered in this order */
-    const allCategoryNames = Object.keys(categoricalSelection).sort();
+    const allCategoryNames = ControlsHelpers.selectableCategoryNames(
+      schema,
+      ControlsHelpers.maxCategoryItems(config)
+    ).sort();

    return (
      <div
@@ -139,20 +143,26 @@ class Categories extends React.Component {
        <AnnoDialog
          isActive={createAnnoModeActive}
          title="Create new category"
-          instruction="New, unique category name:"
+          instruction={this.instruction(newCategoryText)}
          cancelTooltipContent="Close this dialog without creating a category."
          primaryButtonText="Create new category"
+          primaryButtonProps={{ "data-testid": "submit-category" }}
          text={newCategoryText}
          validationError={this.categoryNameError(newCategoryText)}
-          errorMessage={this.categoryNameErrorMessage(newCategoryText)}
          handleSubmit={this.handleCreateUserAnno}
          handleCancel={this.handleDisableAnnoMode}
          annoInput={
-            <AnnoInputs
-              text={newCategoryText}
-              handleItemChange={this.handleSuggestActiveItemChange}
-              handleChoice={this.handleChoice}
-              handleTextChange={this.handleNewCategoryText}
+            <LabelInput
+              labelSuggestions={ontologyEnabled ? ontology.terms : null}
+              onChange={this.handleChange}
+              onSelect={this.handleSelect}
+              inputProps={{
+                "data-testid": "new-category-name",
+                leftIcon: "tag",
+                intent: "none",
+                autoFocus: true
+              }}
+              newLabelMessage="New category"
            />
          }
          annoSelect={
@@ -173,6 +183,8 @@ class Categories extends React.Component {
            <Category
              key={catName}
              metadataField={catName}
+              onExpansionChange={this.onExpansionChange}
+              isExpanded={expandedCats.has(catName)}
              createAnnoModeActive={createAnnoModeActive}
              isUserAnno={false}
            />
@@ -184,6 +196,8 @@ class Categories extends React.Component {
            <Category
              key={catName}
              metadataField={catName}
+              onExpansionChange={this.onExpansionChange}
+              isExpanded={expandedCats.has(catName)}
              createAnnoModeActive={createAnnoModeActive}
              isUserAnno
            />
@@ -191,7 +205,11 @@ class Categories extends React.Component {
        )}
        {writableCategoriesEnabled ? (
          <div>
-            <Button onClick={this.handleEnableAnnoMode} intent="primary">
+            <Button
+              data-testid="open-annotation-dialog"
+              onClick={this.handleEnableAnnoMode}
+              intent="primary"
+            >
              Create new category
            </Button>
          </div>
@@ -2,37 +2,37 @@ import React from "react";
 import _ from "lodash";
 import { connect } from "react-redux";
 import { FaChevronRight, FaChevronDown } from "react-icons/fa";
-import { Button, Tooltip, Icon } from "@blueprintjs/core";
+import { AnchorButton, Button, Tooltip, Icon } from "@blueprintjs/core";
 import CategoryFlipperLayout from "./categoryFlipperLayout";
 import AnnoMenu from "./annoMenuCategory";
 import AnnoDialogEditCategoryName from "./annoDialogEditCategoryName";
 import AnnoDialogAddLabel from "./annoDialogAddLabel";
-import AnnoDialogAddLabelFromOntology from "./annoDialogAddLabelFromOntology";

 import * as globals from "../../globals";
+import maybeTruncateString from "../../util/maybeTruncateString";

@connect(state => ({
  colorAccessor: state.colors.colorAccessor,
  categoricalSelection: state.categoricalSelection,
  annotations: state.annotations,
-  universe: state.universe,
-  ontology: state.ontology,
-  ontologyLoading: state.ontology?.loading,
-  ontologyEnabled: state.ontology?.enabled
+  universe: state.universe
 }))
 class Category extends React.Component {
  constructor(props) {
    super(props);
    this.state = {
-      isChecked: true,
-      isExpanded: false
+      isChecked: true
    };
  }

  componentDidUpdate(prevProps) {
    const { categoricalSelection, metadataField } = this.props;
-    if (categoricalSelection !== prevProps.categoricalSelection) {
-      const cat = categoricalSelection[metadataField];
+    const cat = categoricalSelection?.[metadataField];
+    if (
+      categoricalSelection !== prevProps.categoricalSelection &&
+      !!cat &&
+      !!this.checkbox
+    ) {
      const categoryCount = {
        // total number of categories in this dimension
        totalCatCount: cat.numCategoryValues,
@@ -90,20 +90,80 @@ class Category extends React.Component {
    // || this.checkbox.indeterminate === false
    if (isChecked) {
      this.toggleNone();
-    } else if (!isChecked) {
+    } else {
      this.toggleAll();
    }
  }

+  renderIsStillLoading() {
+    /*
+    We are still loading this category, so render a "busy" signal.
+    */
+    const { metadataField } = this.props;
+    return (
+      <div
+        style={{
+          maxWidth: globals.maxControlsWidth
+        }}
+      >
+        <div
+          style={{
+            display: "flex",
+            justifyContent: "space-between",
+            alignItems: "baseline"
+          }}
+        >
+          <div
+            style={{
+              display: "flex",
+              justifyContent: "flex-start",
+              alignItems: "flex-start"
+            }}
+          >
+            <label className="bp3-control bp3-checkbox">
+              <input disabled checked={true} type="checkbox" />
+              <span className="bp3-control-indicator" />
+            </label>
+            <span
+              style={{
+                cursor: "pointer",
+                display: "inline-block"
+              }}
+            >
+              {metadataField}
+            </span>
+          </div>
+          <div>
+            <Button minimal loading intent="primary" />
+          </div>
+        </div>
+      </div>
+    );
+  }
+
  render() {
-    const { isExpanded, isChecked } = this.state;
+    const { isChecked } = this.state;
    const {
      metadataField,
+      categoricalSelection,
      colorAccessor,
      isUserAnno,
-      annotations
+      annotations,
+      isExpanded,
+      onExpansionChange
    } = this.props;

+    const isStillLoading = !(categoricalSelection?.[metadataField] ?? false);
+    if (isStillLoading) {
+      return this.renderIsStillLoading();
+    }
+
+    const isTruncated = _.get(
+      categoricalSelection,
+      [metadataField, "isTruncated"],
+      false
+    );
+
    return (
      <CategoryFlipperLayout
        metadataField={metadataField}
@@ -120,7 +180,7 @@ class Category extends React.Component {
          <label className="bp3-control bp3-checkbox">
            <input
              data-testclass="category-select"
-              data-testid={`category-select-${metadataField}`}
+              data-testid={`${metadataField}:category-select`}
              onChange={this.handleToggleAllClick.bind(this)}
              ref={el => {
                this.checkbox = el;
@@ -132,7 +192,7 @@ class Category extends React.Component {
            <span className="bp3-control-indicator" />
          </label>
          <span
-            data-testid={`category-expand-${metadataField}`}
+            data-testid={`${metadataField}:category-expand`}
            style={{
              cursor: "pointer",
              display: "inline-block"
@@ -142,14 +202,17 @@ class Category extends React.Component {
                annotations.isEditingCategoryName &&
                annotations.categoryBeingEdited === metadataField;
              if (!editingCategory) {
-                this.setState({ isExpanded: !isExpanded });
+                onExpansionChange(metadataField);
              }
            }}
          >
            {isUserAnno ? (
              <Icon style={{ marginRight: 5 }} icon="tag" iconSize={16} />
            ) : null}
-            {metadataField}
+            {maybeTruncateString(
+              metadataField,
+              globals.categoryDisplayStringMaxLength
+            )}
            {isExpanded ? (
              <FaChevronDown
                data-testclass="category-expand-is-expanded"
@@ -165,28 +228,32 @@ class Category extends React.Component {
        </div>
        {<AnnoDialogEditCategoryName metadataField={metadataField} />}
        {<AnnoDialogAddLabel metadataField={metadataField} />}
-        {<AnnoDialogAddLabelFromOntology metadataField={metadataField} />}
        <div>
          <AnnoMenu
            metadataField={metadataField}
            isUserAnno={isUserAnno}
            createText="Add a new label to this category"
-            createFromOntologyText="Add a new label to this category using existing ontology terms"
            editText="Edit this category's name"
            deleteText="Delete this category, all associated labels, and remove all cell assignments"
          />

          <Tooltip
-            content="Use as color scale"
+            content={
+              isTruncated
+                ? `Coloring by ${metadataField} is disabled, as it exceeds the limit of ${globals.maxCategoricalOptionsToDisplay} labels`
+                : "Use as color scale"
+            }
            position="bottom"
+            usePortal={false}
            hoverOpenDelay={globals.tooltipHoverOpenDelay}
          >
-            <Button
+            <AnchorButton
              data-testclass="colorby"
              data-testid={`colorby-${metadataField}`}
              onClick={this.handleColorChange}
              active={colorAccessor === metadataField}
              intent={colorAccessor === metadataField ? "primary" : "none"}
+              disabled={isTruncated}
              icon="tint"
            />
          </Tooltip>
@@ -0,0 +1,176 @@
+import React from "react";
+import { InputGroup, MenuItem, Keys } from "@blueprintjs/core";
+import { Suggest } from "@blueprintjs/select";
+import fuzzysort from "fuzzysort";
+
+export default class LabelInput extends React.PureComponent {
+  /*
+  Input widget for text labels, which acts like an InputGroup, but will also 
+  accept a suggestion list (of labels), with sublime-like suggest search.
+
+  Properties:
+
+  * labelSuggestions -- array of suggested lables. May be array of string or
+    other objects. If array of objects, specify `labelKey`. If null, suggestion
+    mode is disabled.
+  * onSelect -- optional, callback upon selection of item from labelSuggestions.
+    (label) => void
+  * onChange -- optional, callback upon change in text input.  (label) => void
+  * label -- component value, for controlled use
+  * newLabelMessage -- text to display when user enters a label not in labelSuggestions
+    (not used if suggestion mode disabled)
+  * inputProps -- will be passed to InputGroup
+  * popoverProps -- will be passed to <Suggest>
+  */
+
+  constructor(props) {
+    super(props);
+
+    const { label } = props;
+    const query = label || "";
+    const queryResults = this.filterLabels(query);
+    this.state = {
+      query,
+      queryResults
+    };
+  }
+
+  handleQueryChange = (query, event) => {
+    // https://github.com/palantir/blueprint/issues/2983
+    if (!event) return;
+
+    const queryResults = this.filterLabels(query);
+    this.setState({
+      query,
+      queryResults
+    });
+
+    const { onChange } = this.props;
+    if (onChange) onChange(query, event);
+  };
+
+  handleItemSelect = (item, event) => {
+    /* only report the select if not already reported via onChange() */
+    const { target } = item;
+    const { query } = this.state;
+    const { onSelect } = this.props;
+    if (target !== query && onSelect) onSelect(target, event);
+  };
+
+  handleKeyDown = e => {
+    /* 
+    prevent these events from propagating to containing form/dialog
+    and causing further side effects (eg, closing dialog, submitting
+    form, etc).
+    */
+    const { keyCode } = e;
+    if (keyCode === Keys.ENTER || keyCode === Keys.ESCAPE) {
+      e.preventDefault();
+    }
+    if (keyCode === Keys.ESCAPE) {
+      e.stopPropagation();
+    }
+  };
+
+  handleChange = e => {
+    const { onChange } = this.props;
+    if (onChange) onChange(e.target.value);
+  };
+
+  renderLabelSuggestion = (queryResult, { handleClick, modifiers }) => {
+    if (queryResult.newLabel) {
+      const { newLabelMessage } = this.props;
+      return (
+        <MenuItem
+          icon="flag"
+          active={modifiers.active}
+          disabled={modifiers.disabled}
+          key={queryResult.target}
+          onClick={handleClick}
+          text={<em>{queryResult.target}</em>}
+          label={newLabelMessage || "New label"}
+        />
+      );
+    }
+    return (
+      <MenuItem
+        active={modifiers.active}
+        disabled={modifiers.disabled}
+        key={queryResult.target}
+        onClick={handleClick}
+        text={queryResult.target}
+      />
+    );
+  };
+
+  /* maxinum number of suggestions */
+  static QueryResultLimit = 100;
+
+  filterLabels(query) {
+    const { labelSuggestions } = this.props;
+    if (!labelSuggestions) return [];
+
+    /* empty query is wildcard */
+    if (query === "") {
+      return labelSuggestions.slice(0, LabelInput.QueryResultLimit).map(l => ({
+        target: l,
+        score: -10000
+      }));
+    }
+
+    /* else, do a fuzzy query */
+    const options = {
+      limit: LabelInput.QueryResultLimit,
+      threshold: -10000 // don't return bad results
+    };
+    let queryResults = fuzzysort.go(query, labelSuggestions, options);
+    /* exact match will always be first in list */
+    if (query !== "" && queryResults[0]?.target !== query)
+      queryResults = [{ target: query, newLabel: true }, ...queryResults];
+
+    return queryResults;
+  }
+
+  render() {
+    const { props } = this;
+    const { labelSuggestions, label, autoFocus = true } = props;
+    const suggestEnabled = !!labelSuggestions && labelSuggestions.length > 0;
+
+    if (!suggestEnabled) {
+      return (
+        <InputGroup
+          autoFocus={autoFocus}
+          {...props.inputProps} // eslint-disable-line react/jsx-props-no-spreading
+          value={label}
+          onChange={this.handleChange}
+        />
+      );
+    }
+
+    const popoverProps = {
+      minimal: true,
+      ...props.popoverProps
+    };
+    const inputProps = {
+      ...props.inputProps,
+      autoFocus: false
+    };
+    const { queryResults } = this.state;
+    return (
+      <>
+        <Suggest
+          fill
+          inputValueRenderer={i => i.target}
+          items={queryResults}
+          itemRenderer={this.renderLabelSuggestion}
+          onItemSelect={this.handleItemSelect}
+          query={label}
+          onQueryChange={this.handleQueryChange}
+          popoverProps={popoverProps}
+          inputProps={inputProps}
+          onKeyDown={this.handleKeyDown}
+        />
+      </>
+    );
+  }
+}
@@ -1,4 +1,6 @@
 import React from "react";
+import { Colors } from "@blueprintjs/core";
+
 import { AnnotationsHelpers } from "../../util/stateManager";

 export function isLabelErroneous(label, metadataField, ontology, schema) {
@@ -24,34 +26,38 @@ export function isLabelErroneous(label, metadataField, ontology, schema) {
 	return false;
 }

-export function labelErrorMessage(label, metadataField, ontology, schema) {
-	const err = isLabelErroneous(label, metadataField, ontology, schema);
+/* all other errors - map code to human error message */
+const errorMessageMap = {
+	"empty-string": "Blank names not allowed",
+	duplicate: "Name must be unique",
+	"trim-spaces": "Leading and trailing spaces not allowed",
+	"illegal-characters":
+		"Only alphanumeric and special characters (-_.) allowed",
+	"multi-space-run": "Multiple consecutive spaces not allowed"
+};

-	if (err === "duplicate") {
-		/* duplicate error is special cased because it has special formatting */
-		return (
-			<span>
-				<span style={{ fontStyle: "italic" }}>{label}</span> already
-				exists already exists within{" "}
-				<span style={{ fontStyle: "italic" }}>{metadataField}</span>{" "}
+export function labelPrompt(err, prolog, epilog) {
+	let errPrompt = null;
+	if (err) {
+		let errMsg = errorMessageMap[err] ?? "error";
+		errMsg = errMsg[0].toLowerCase() + errMsg.slice(1);
+		errPrompt = (
+			<span
+				style={{
+					marginTop: 7,
+					color: Colors.ORANGE3
+				}}
+			>
+				{errMsg}
 			</span>
 		);
 	}
-
-	if (err) {
-		/* all other errors - map code to human error message */
-		const errorMessageMap = {
-			"empty-string": "Blank names not allowed",
-			duplicate: "Name must be unique",
-			"trim-spaces": "Leading and trailing spaces not allowed",
-			"illegal-characters":
-				"Only alphanumeric and special characters (-_.) allowed",
-			"multi-space-run": "Multiple consecutive spaces not allowed"
-		};
-		const errorMessage = errorMessageMap[err] ?? "error";
-		return <span>{errorMessage}</span>;
-	}
-
-	/* no error, no message generated */
-	return null;
+	return (
+		<span>
+			{prolog}
+			{err ? " - " : null}
+			{errPrompt}
+			{epilog}
+		</span>
+	);
 }
@@ -1,57 +0,0 @@
-import React from "react";
-import { connect } from "react-redux";
-import { Button, MenuItem } from "@blueprintjs/core";
-import { Select } from "@blueprintjs/select";
-import fuzzysort from "fuzzysort";
-
-const filterOntology = (query, ontology) =>
-  /* fires on load, once, and then for each character typed into the input */
-  fuzzysort.go(query, ontology, {
-    limit: 100,
-    threshold: -10000 // don't return bad results
-  });
-
-@connect()
-class ChooseOntologySelect extends React.Component {
-  constructor(props) {
-    super(props);
-    this.state = {};
-  }
-
-  render() {
-    const {
-      ontology,
-      categoryToDuplicate,
-      handleChooseOntologyTermFromDropdown
-    } = this.props;
-    return (
-      <div>
-        <Select
-          items={
-            ontology?.terms ||
-            [] /* this is a placeholder, could be  a subcomponent to avoid this */
-          }
-          filterable
-          itemListPredicate={filterOntology}
-          itemRenderer={(d, { handleClick }) => {
-            return (
-              <MenuItem onClick={handleClick} key={d.target} text={d.target} />
-            );
-          }}
-          noResults={<MenuItem disabled text="No results." />}
-          onItemSelect={d => {
-            handleChooseOntologyTermFromDropdown(d);
-          }}
-        >
-          {/* children become the popover target; render value here */}
-          <Button
-            text={categoryToDuplicate || "Choose an Ontology Term"}
-            rightIcon="double-caret-vertical"
-          />
-        </Select>
-      </div>
-    );
-  }
-}
-
-export default ChooseOntologySelect;
@@ -1,4 +1,3 @@
-// jshint esversion: 6
 import { connect } from "react-redux";
 import React from "react";

@@ -9,40 +8,49 @@ import {
  Popover,
  Position,
  PopoverInteractionKind,
-  Tooltip,
-  Colors
+  Tooltip
 } from "@blueprintjs/core";
 import Occupancy from "./occupancy";
 import * as globals from "../../globals";
 import styles from "./categorical.css";
 import AnnoDialog from "./annoDialog";
-import AnnoInputs from "./annoInputs";
+import LabelInput from "./labelInput";

 import { AnnotationsHelpers } from "../../util/stateManager";
-import { labelErrorMessage, isLabelErroneous } from "./labelUtil";
+import { labelPrompt, isLabelErroneous } from "./labelUtil";

-@connect(state => ({
-  categoricalSelection: state.categoricalSelection,
-  annotations: state.annotations,
-  colorScale: state.colors.scale,
-  colorAccessor: state.colors.colorAccessor,
-  pointDilation: state.pointDilation,
-  schema: state.world?.schema,
-  world: state.world,
-  crossfilter: state.crossfilter,
-  ontology: state.ontology,
-  ontologyLoading: state.ontology?.loading,
-  ontologyEnabled: state.ontology?.enabled
-}))
+/* this is defined outside of the class so we can use it in connect() */
+function _currentLabel(ownProps, categoricalSelection) {
+  const { metadataField, categoryIndex } = ownProps;
+  return String(
+    categoricalSelection[metadataField].categoryValues[categoryIndex]
+  ).valueOf();
+}
+
+@connect((state, ownProps) => {
+  const { pointDilation, categoricalSelection } = state;
+  const { metadataField, categoryField } = ownProps;
+  const isDilated =
+    pointDilation.metadataField === metadataField &&
+    pointDilation.categoryField ===
+      _currentLabel(ownProps, categoricalSelection);
+  return {
+    categoricalSelection,
+    annotations: state.annotations,
+    colorScale: state.colors.scale,
+    colorAccessor: state.colors.colorAccessor,
+    schema: state.world?.schema,
+    world: state.world,
+    crossfilter: state.crossfilter,
+    ontology: state.ontology,
+    isDilated
+  };
+})
 class CategoryValue extends React.Component {
  constructor(props) {
    super(props);
    this.state = {
-      editedLabelText: String(
-        props.categoricalSelection[props.metadataField].categoryValues[
-          props.categoryIndex
-        ]
-      ).valueOf()
+      editedLabelText: this.currentLabel()
    };
  }

@@ -54,9 +62,7 @@ class CategoryValue extends React.Component {
      prevProps.categoryIndex !== categoryIndex
    ) {
      this.setState({
-        editedLabelText: String(
-          categoricalSelection[metadataField].categoryValues[categoryIndex]
-        ).valueOf()
+        editedLabelText: this.currentLabel()
      });
    }
  }
@@ -83,10 +89,11 @@ class CategoryValue extends React.Component {
    });
  };

-  handleEditValue = () => {
+  handleEditValue = e => {
    const { dispatch, metadataField, categoryIndex } = this.props;
    const { editedLabelText } = this.state;
    const label = this.getLabel();
+    this.cancelEditMode();
    dispatch({
      type: "annotation: label edited",
      editedLabel: editedLabelText,
@@ -94,51 +101,30 @@ class CategoryValue extends React.Component {
      categoryIndex,
      label
    });
+    e.preventDefault();
  };

-  handleCreateArbitraryLabel = editedLabelTextNotInOntology => {
+  handleCreateArbitraryLabel = txt => {
    const { dispatch, metadataField, categoryIndex } = this.props;
    const label = this.getLabel();
-
+    this.cancelEditMode();
    dispatch({
      type: "annotation: label edited",
      metadataField,
-      editedLabel: editedLabelTextNotInOntology,
+      editedLabel: txt,
      categoryIndex,
      label
    });
  };

  labelNameError = name => {
-    const {
-      metadataField,
-      ontology,
-      schema,
-      categoricalSelection,
-      categoryIndex
-    } = this.props;
-    const category = categoricalSelection[metadataField];
-    const displayString = String(
-      category.categoryValues[categoryIndex]
-    ).valueOf();
-    if (name === displayString) return false;
+    const { metadataField, ontology, schema } = this.props;
+    if (name === this.currentLabel()) return false;
    return isLabelErroneous(name, metadataField, ontology, schema);
  };

-  labelNameErrorMessage = name => {
-    const {
-      metadataField,
-      ontology,
-      schema,
-      categoricalSelection,
-      categoryIndex
-    } = this.props;
-    const category = categoricalSelection[metadataField];
-    const displayString = String(
-      category.categoryValues[categoryIndex]
-    ).valueOf();
-    if (name === displayString) return null;
-    return labelErrorMessage(name, metadataField, ontology, schema);
+  instruction = label => {
+    return labelPrompt(this.labelNameError(label), "New, unique label", ":");
  };

  activateEditLabelMode = () => {
@@ -150,8 +136,11 @@ class CategoryValue extends React.Component {
    });
  };

-  cancelEdit = () => {
+  cancelEditMode = () => {
    const { dispatch, metadataField, categoryIndex } = this.props;
+    this.setState({
+      editedLabelText: this.currentLabel()
+    });
    dispatch({
      type: "annotation: cancel edit label mode",
      metadataField,
@@ -196,7 +185,7 @@ class CategoryValue extends React.Component {
    const crossfilterChange =
      props.isUserAnno && props.crossfilter !== nextProps.crossfilter;
    const editingLabel = state.editedLabelText !== nextState.editedLabelText;
-    const dilationChange = props.pointDilation !== nextProps.pointDilation;
+    const dilationChange = props.isDilated !== nextProps.isDilated;

    return (
      valueSelectionChange ||
@@ -253,6 +242,11 @@ class CategoryValue extends React.Component {
    return label;
  };

+  currentLabel() {
+    const { categoricalSelection } = this.props;
+    return _currentLabel(this.props, categoricalSelection);
+  }
+
  isAddCurrentSelectionDisabled(category, value) {
    /*
    disable "add current selection to label", if one of the following is true:
@@ -292,12 +286,13 @@ class CategoryValue extends React.Component {
      schema,
      isUserAnno,
      annotations,
-      ontologyEnabled,
+      ontology,
      // flippedProps is potentially brittle, their docs want {...flippedProps} on our div,
      // our lint doesn't like jsx spread, we are version pinned to prevent api change on their part
      flippedProps,
-      pointDilation
+      isDilated
    } = this.props;
+    const ontologyEnabled = ontology?.enabled ?? false;

    const { editedLabelText } = this.state;

@@ -307,9 +302,7 @@ class CategoryValue extends React.Component {
    const selected = category.categoryValueSelected[categoryIndex];
    const count = category.categoryValueCounts[categoryIndex];
    const value = category.categoryValues[categoryIndex];
-    const displayString = String(
-      category.categoryValues[categoryIndex]
-    ).valueOf();
+    const displayString = this.currentLabel();

    /* this is the color scale, so add swatches below */
    const isColorBy = metadataField === colorAccessor;
@@ -357,12 +350,7 @@ class CategoryValue extends React.Component {
        data-portal-key={flippedProps["data-portal-key"]}
        className={
          /* This code is to change the styles on centroid label hover is causing over-rendering */
-          `${styles.value}${
-            pointDilation.metadataField === metadataField &&
-            pointDilation.categoryField === displayString
-              ? ` ${styles.hover}`
-              : ""
-          }`
+          `${styles.value}${isDilated ? ` ${styles.hover}` : ""}`
        }
        data-testclass="categorical-row"
        style={{
@@ -433,28 +421,33 @@ class CategoryValue extends React.Component {
              <div>
                <AnnoDialog
                  isActive={editModeActive}
+                  inputProps={{
+                    "data-testid": `${metadataField}:edit-label-name-dialog`
+                  }}
+                  primaryButtonProps={{
+                    "data-testid": `${metadataField}:${displayString}:submit-label-edit`
+                  }}
                  title="Edit label"
-                  instruction={`New label text must be unique within category ${metadataField}:`}
+                  instruction={this.instruction(editedLabelText)}
                  cancelTooltipContent="Close this dialog without editing label text."
                  primaryButtonText="Change label text"
                  text={editedLabelText}
                  categoryToDuplicate={null}
                  validationError={this.labelNameError(editedLabelText)}
-                  errorMessage={this.labelNameErrorMessage(editedLabelText)}
                  handleSubmit={this.handleEditValue}
-                  handleCancel={this.cancelEdit}
+                  handleCancel={this.cancelEditMode}
                  annoInput={
-                    <AnnoInputs
-                      useSuggest={ontologyEnabled}
-                      text={editedLabelText}
-                      handleCreateArbitraryLabel={
-                        this.handleCreateArbitraryLabel
-                      }
-                      handleItemChange={this.handleSuggestActiveItemChange}
-                      handleChoice={this.handleChoice}
-                      handleTextChange={this.handleTextChange}
-                      isTextInvalid={this.labelNameError}
-                      isTextInvalidErrorMessage={this.labelNameErrorMessage}
+                    <LabelInput
+                      label={editedLabelText}
+                      labelSuggestions={ontologyEnabled ? ontology.terms : null}
+                      onChange={this.handleTextChange}
+                      onSelect={this.handleTextChange}
+                      inputProps={{
+                        "data-testid": `${metadataField}:${displayString}:edit-label-name`,
+                        leftIcon: "tag",
+                        intent: "none",
+                        autoFocus: true
+                      }}
                    />
                  }
                  annoSelect={null}
@@ -464,7 +457,10 @@ class CategoryValue extends React.Component {
          </div>
          <span style={{ flexShrink: 0 }}>
            {colorAccessor && !isColorBy && !annotations.isEditingLabelName ? (
-              <Occupancy category={category} {...this.props} />
+              <Occupancy
+                category={category}
+                {...this.props} // eslint-disable-line react/jsx-props-no-spreading
+              />
            ) : null}
          </span>
        </div>
@@ -513,7 +509,7 @@ class CategoryValue extends React.Component {
                      <MenuItem
                        icon="plus"
                        data-testclass="handleAddCurrentSelectionToThisLabel"
-                        data-testid={`handleAddCurrentSelectionToThisLabel-${metadataField}`}
+                        data-testid={`${metadataField}:${displayString}:add-current-selection-to-this-label`}
                        onClick={this.handleAddCurrentSelectionToThisLabel}
                        text={
                          <span>
@@ -541,7 +537,7 @@ class CategoryValue extends React.Component {
                          icon="edit"
                          text="Edit this label's name"
                          data-testclass="handleEditValue"
-                          data-testid={`handleEditValue-${metadataField}`}
+                          data-testid={`${metadataField}:${displayString}:edit-label`}
                          onClick={this.activateEditLabelMode}
                          disabled={annotations.isEditingLabelName}
                        />
@@ -551,7 +547,7 @@ class CategoryValue extends React.Component {
                          icon="delete"
                          intent="danger"
                          data-testclass="handleDeleteValue"
-                          data-testid={`handleDeleteValue-${metadataField}`}
+                          data-testid={`${metadataField}:${displayString}:delete-label`}
                          onClick={this.handleDeleteValue}
                          text={`Delete this label, and reassign all cells to type '${globals.unassignedCategoryLabel}'`}
                        />
@@ -567,7 +563,7 @@ class CategoryValue extends React.Component {
                      minHeight: 16
                    }}
                    data-testclass="seeActions"
-                    data-testid={`seeActions-${metadataField}`}
+                    data-testid={`${metadataField}:${displayString}:see-actions`}
                    icon="more"
                    small
                    minimal
@@ -2,10 +2,10 @@
 /* rc slider https://www.npmjs.com/package/rc-slider */

 import React from "react";
-import _ from "lodash";
 import { connect } from "react-redux";
-import * as globals from "../../globals";
+import { Button } from "@blueprintjs/core";

+import * as globals from "../../globals";
 import HistogramBrush from "../brushableHistogram";

@connect(state => ({
@@ -14,17 +14,7 @@ import HistogramBrush from "../brushableHistogram";
  colorScale: state.colors.scale,
  schema: state.world?.schema
 }))
-class Continuous extends React.Component {
-  constructor(props) {
-    super(props);
-    this.hasContinuous = false;
-    this.continuousChecked = false;
-
-    this.state = {};
-  }
-
-  componentDidUpdate() {}
-
+class Continuous extends React.PureComponent {
  handleColorAction = key => {
    return () => {
      const { dispatch, obsAnnotations } = this.props;
@@ -37,61 +27,84 @@ class Continuous extends React.Component {
    };
  };

+  static renderIsStillLoading(zebra, key) {
+    return (
+      <div
+        key={key}
+        style={{
+          padding: globals.leftSidebarSectionPadding,
+          backgroundColor: zebra % 2 === 0 ? globals.lightestGrey : "white"
+        }}
+      >
+        <div
+          style={{
+            display: "flex",
+            justifyContent: "space-between",
+            justifyItems: "center",
+            alignItems: "center"
+          }}
+        >
+          <div style={{ minWidth: 30 }} />
+          <div style={{ display: "flex", alignSelf: "center" }}>
+            <span style={{ fontStyle: "italic" }}>{key}</span>
+          </div>
+          <div
+            style={{
+              display: "flex",
+              justifyContent: "flex-end"
+            }}
+          >
+            <Button minimal loading intent="primary" />
+          </div>
+        </div>
+      </div>
+    );
+  }
+
  render() {
    const { obsAnnotations, schema } = this.props;
-    if (schema && !this.continuousChecked) {
-      this.hasContinuous = _.some(
-        schema.annotations.obs,
-        d => d.type === "int32" || d.type === "float32"
-      );
-      this.continuousChecked = true; /* only do this once */
-    }
+
+    const obsIndex = schema.annotations.obs.index;
+    const allContinuousNames = schema.annotations.obs.columns
+      .filter(col => col.type === "int32" || col.type === "float32")
+      .filter(col => col.name !== obsIndex)
+      .map(col => col.name);

    /* initial value for iterator to simulate index, ranges is an object */
    let zebra = 0;

    return (
      <div>
-        {this.hasContinuous ? (
-          <p
-            style={{
-              ...globals.leftSidebarSectionHeading,
-              marginTop: 40,
-              paddingLeft: globals.leftSidebarSectionPadding
-            }}
-          >
-            Continuous metadata
-          </p>
-        ) : null}
-        {obsAnnotations
-          ? _.map(obsAnnotations.colIndex.keys(), key => {
-              const isColorField =
-                key.includes("color") || key.includes("Color");
-              if (key === schema.annotations.obs.index || isColorField)
-                return null;
+        {allContinuousNames.map(key => {
+          if (!obsAnnotations.hasCol(key)) {
+            // still loading!
+            zebra += 1;
+            return Continuous.renderIsStillLoading(zebra, key);
+          }

-              const summary = obsAnnotations.col(key).summarize();
-              const nonFiniteExtent =
-                summary.min === undefined ||
-                summary.max === undefined ||
-                Number.isNaN(summary.min) ||
-                Number.isNaN(summary.max);
-              if (!summary.categorical && !nonFiniteExtent) {
-                zebra += 1;
-                return (
-                  <HistogramBrush
-                    key={key}
-                    field={key}
-                    isObs
-                    zebra={zebra % 2 === 0}
-                    ranges={summary}
-                    handleColorAction={this.handleColorAction(key)}
-                  />
-                );
-              }
-              return null;
-            })
-          : null}
+          // data loaded and available
+          const summary = obsAnnotations.col(key).summarize();
+          const nonFiniteExtent =
+            summary.min === undefined ||
+            summary.max === undefined ||
+            Number.isNaN(summary.min) ||
+            Number.isNaN(summary.max);
+          if (!summary.categorical && !nonFiniteExtent) {
+            zebra += 1;
+            return (
+              <HistogramBrush
+                key={key}
+                field={key}
+                isObs
+                zebra={zebra % 2 === 0}
+                ranges={summary}
+                handleColorAction={this.handleColorAction(key)}
+              />
+            );
+          }
+
+          return null;
+        })}
      </div>
    );
  }
@@ -2,7 +2,7 @@ import { Position, Toaster, Intent } from "@blueprintjs/core";

 /** Singleton toaster instance. Create separate instances for different options. */

-const ErrorToastTopCenter = Toaster.create({
+const ToastTopCenter = Toaster.create({
  className: "recipe-toaster",
  position: Position.TOP
 });
@@ -11,21 +11,38 @@ const ErrorToastTopCenter = Toaster.create({
 A "user" error - eg, bad input
 */
 export const postUserErrorToast = message =>
-  ErrorToastTopCenter.show({ message, intent: Intent.WARNING });
+  ToastTopCenter.show({ message, intent: Intent.WARNING });

 /*
 A toast the user must dismiss manually, because they need to act on its information,
 ie., 8 bulk add genes out of 40 were bad. Manually see which ones and fix.
 */
 export const keepAroundErrorToast = message =>
-  ErrorToastTopCenter.show({ message, timeout: 0, intent: Intent.WARNING });
+  ToastTopCenter.show({ message, timeout: 0, intent: Intent.WARNING });

 /*
 a hard network error
 */
 export const postNetworkErrorToast = message =>
-  ErrorToastTopCenter.show({
+  ToastTopCenter.show({
    message,
    timeout: 30000,
    intent: Intent.DANGER
  });
+
+/*
+Async message to user
+*/
+export const postAsyncSuccessToast = message =>
+  ToastTopCenter.show({
+    message,
+    timeout: 10000,
+    intent: Intent.SUCCESS
+  });
+
+export const postAsyncFailureToast = message =>
+  ToastTopCenter.show({
+    message,
+    timeout: 10000,
+    intent: Intent.WARNING
+  });
@@ -103,7 +103,8 @@ class GeneExpression extends React.Component {
      if (genes.length === 0) {
        return keepAroundErrorToast("Must enter a gene name.");
      }
-      const worldGenes = world.varAnnotations.col(varIndexName).asArray();
+      const worldGenes =
+        world.varAnnotations?.col(varIndexName)?.asArray() || [];

      // These gene lists are unique enough where memoization is useless
      const upperGenes = this._genesToUpper(genes);
@@ -181,9 +182,9 @@ class GeneExpression extends React.Component {
    const gene = g.target;
    if (userDefinedGenes.indexOf(gene) !== -1) {
      postUserErrorToast("That gene already exists");
-    } else if (userDefinedGenes.length > 15) {
+    } else if (userDefinedGenes.length > globals.maxUserDefinedGenes) {
      postUserErrorToast(
-        "That's too many genes, you can have at most 15 user defined genes"
+        `That's too many genes, you can have at most ${globals.maxUserDefinedGenes} user defined genes`
      );
    } else if (
      world.varAnnotations.col(varIndexName).indexOf(gene) === undefined
@@ -206,8 +207,12 @@ class GeneExpression extends React.Component {
      differential
    } = this.props;
    const varIndexName = world?.schema?.annotations?.var?.index;
+    const varIndex = world?.varAnnotations?.col(varIndexName)?.asArray();
    const { tab, bulkAdd, activeItem } = this.state;

+    // may still be loading!
+    if (!varIndex) return null;
+
    return (
      <div>
        <div>
@@ -268,11 +273,7 @@ class GeneExpression extends React.Component {
                itemListPredicate={filterGenes}
                onActiveItemChange={item => this.setState({ activeItem: item })}
                itemRenderer={renderGene.bind(this)}
-                items={
-                  world && world.varAnnotations
-                    ? world.varAnnotations.col(varIndexName).asArray()
-                    : ["No genes"]
-                }
+                items={varIndex || ["No genes"]}
                popoverProps={{ minimal: true }}
              />
              <Button
@@ -255,7 +255,7 @@ class Graph extends React.Component {
    const { regl, toolSVG, camera, modelTF } = this.state;
    let stateChanges = {};

-    if (regl && world) {
+    if (regl && world && crossfilter) {
      /* update the regl and point rendering state */
      const { obsLayout, nObs } = world;
      const { drawPoints, pointBuffer, colorBuffer, flagBuffer } = this.state;
@@ -398,7 +398,7 @@ class Graph extends React.Component {
      // don't return "change" of state unless we are really changing it!
      const { toolSVG } = this.state;
      if (toolSVG === undefined) return {};
-      else return { toolSVG: undefined };
+      return { toolSVG: undefined };
    }

    let handleStart;
@@ -422,8 +422,7 @@ class Graph extends React.Component {
      handleEnd,
      handleCancel,
      responsive,
-      this.graphPaddingRightLeft,
-      graphInteractionMode
+      this.graphPaddingRightLeft
    );

    return { toolSVG: newToolSVG, tool, container };
@@ -749,6 +748,11 @@ class Graph extends React.Component {
              graphPaddingRightLeft={this.graphPaddingRightLeft}
              graphPaddingTop={this.graphPaddingTop}
              responsive={responsive}
+              handleCanvasEvent={
+                graphInteractionMode === "zoom"
+                  ? this.handleCanvasEvent
+                  : undefined
+              }
            >
              <CentroidLabels />
            </GraphOverlayLayer>
@@ -1,7 +1,10 @@
+/* eslint-disable max-classes-per-file */
 /* eslint-disable jsx-a11y/mouse-events-have-key-events */
 import React, { PureComponent } from "react";
 import { connect } from "react-redux";

+import { categoryLabelDisplayStringLongLength } from "../../../globals";
+
 export default
@connect(state => ({
  colorAccessor: state.colors.colorAccessor,
@@ -9,6 +12,21 @@ export default
  labels: state.centroidLabels.labels
 }))
 class CentroidLabels extends PureComponent {
+  // Check to see if centroids have either just been displayed or removed from the overlay
+  componentDidUpdate = prevProps => {
+    const { labels, overlayToggled } = this.props;
+    const prevSize = prevProps.labels.size;
+    const { size } = labels;
+
+    const displayChangeOff = prevSize > 0 && size === undefined;
+    const displayChangeOn = prevSize === undefined && size > 0;
+
+    if (displayChangeOn || displayChangeOff) {
+      // Notify overlay layer of display change
+      overlayToggled("centroidLabels", displayChangeOn);
+    }
+  };
+
  render() {
    const {
      labels,
@@ -28,12 +46,24 @@ class CentroidLabels extends PureComponent {
        fontSize = "18px";
        fontWeight = "800";
      }
+
+      // Mirror LSB middle truncation
+      let label = key;
+      if (label.length > categoryLabelDisplayStringLongLength) {
+        label = `${key.slice(
+          0,
+          categoryLabelDisplayStringLongLength / 2
+        )}…${key.slice(-categoryLabelDisplayStringLongLength / 2)}`;
+      }
+
      labelSVGS.push(
        <g
          // eslint-disable-next-line react/no-array-index-key
          key={key}
          className="centroid-label"
          transform={`translate(${value[0]}, ${value[1]})`}
+          data-testclass="centroid-label"
+          data-testid={`${key}-centroid-label`}
        >
          <text
            transform={inverseTransform}
@@ -62,7 +92,7 @@ class CentroidLabels extends PureComponent {
            }
            pointerEvents="visiblePainted"
          >
-            {key.length > 20 ? `${key.substr(0, 20)}...` : key}
+            {label}
          </text>
        </g>
      );
@@ -12,8 +12,16 @@ class GraphOverlayLayer extends PureComponent {
    This component takes its children (assumed in the data coordinate space ([0, 1] range, origin in bottom left corner))
    and transforms itself multiple times resulting in screen space ([0, screenWidth/Height] range, origin in top left corner)

-    Children are assigned in the graph component
+    Children are assigned in the graph component and must implement onDisplayChange()
   */
+  constructor(props) {
+    super(props);
+
+    this.state = {
+      display: {}
+    };
+  }
+
  matrixToTransformString = m => {
    /* 
      Translates the gl-matrix mat3 to SVG matrix transform style
@@ -30,6 +38,13 @@ class GraphOverlayLayer extends PureComponent {
    return `matrix(${1 / m[0]} 0 0 ${1 / m[4]} 0 0)`;
  };

+  // This is passed to all children, should be called when an overlay's display state is toggled along with the overlay name and its new display state in boolean form
+  overlayToggled = (overlay, displaying) => {
+    this.setState(state => {
+      return { ...state, display: { ...state.display, [overlay]: displaying } };
+    });
+  };
+
  render() {
    const {
      cameraTF,
@@ -38,11 +53,15 @@ class GraphOverlayLayer extends PureComponent {
      responsive,
      graphPaddingRightLeft,
      graphPaddingTop,
-      children
+      children,
+      handleCanvasEvent
    } = this.props;

    if (!cameraTF) return null;

+    const { display } = this.state;
+    const displaying = Object.values(display).some(value => value); // check to see if at least one overlay is currently displayed
+
    const inverseTransform = `${this.reverseMatrixScaleTransformString(
      modelTF
    )} ${this.reverseMatrixScaleTransformString(
@@ -53,7 +72,13 @@ class GraphOverlayLayer extends PureComponent {
      -(responsive.height - graphPaddingTop)}) scale(2 1) scale(${1 /
      (responsive.width - graphPaddingRightLeft)} 1)`;

-    const newChildren = React.Children.toArray(children);
+    // Copy the children passed with the overlay and add the inverse transform and onDisplayChange props
+    const newChildren = React.Children.map(children, child =>
+      cloneElement(child, {
+        inverseTransform,
+        overlayToggled: this.overlayToggled
+      })
+    );

    return (
      <svg
@@ -61,7 +86,12 @@ class GraphOverlayLayer extends PureComponent {
        width={responsive.width - graphPaddingRightLeft}
        height={responsive.height}
        pointerEvents="none"
-        style={{ zIndex: 99 }}
+        style={{
+          zIndex: 99,
+          backgroundColor: displaying ? "rgba(255, 255, 255, 0.55)" : ""
+        }}
+        onMouseMove={handleCanvasEvent}
+        onWheel={handleCanvasEvent}
      >
        <g
          id="canvas-transformation-group-x"
@@ -86,9 +116,7 @@ class GraphOverlayLayer extends PureComponent {
                  id="model-transformation-group"
                  transform={this.matrixToTransformString(modelTF)}
                >
-                  {newChildren.map(child =>
-                    cloneElement(child, { inverseTransform })
-                  )}
+                  {newChildren}
                </g>
              </g>
            </g>
@@ -116,7 +116,7 @@ function Clip(props) {
              <Button
                type="button"
                data-testid="clip-commit"
-                className="bp3-button"
+                intent="primary"
                disabled={isClipDisabled()}
                style={{
                  cursor: "pointer",
@@ -0,0 +1,122 @@
+import React from "react";
+import {
+  AnchorButton,
+  ButtonGroup,
+  Popover,
+  Button,
+  Radio,
+  RadioGroup,
+  Tooltip,
+  Position
+} from "@blueprintjs/core";
+import { connect } from "react-redux";
+import * as globals from "../../globals";
+import { World } from "../../util/stateManager";
+import actions from "../../actions";
+
+@connect(state => ({
+  universe: state.universe,
+  world: state.world,
+  layoutChoice: state.layoutChoice,
+  reembedController: state.reembedController,
+  enableReembedding: state.config?.parameters?.["enable-reembedding"] ?? false
+}))
+class Embedding extends React.PureComponent {
+  handleLayoutChoiceChange = e => {
+    const { dispatch } = this.props;
+    dispatch({
+      type: "set layout choice",
+      layoutChoice: e.currentTarget.value
+    });
+  };
+
+  renderReembedding() {
+    const {
+      enableReembedding,
+      world,
+      universe,
+      dispatch,
+      reembedController
+    } = this.props;
+
+    if (!enableReembedding) return null;
+
+    const loading = !!reembedController?.pendingFetch;
+    const disabled = World.worldEqUniverse(world, universe);
+    const tipContent = disabled
+      ? "Subset cells first, then click to recompute UMAP embedding."
+      : "Click to recompute UMAP embedding on the current cell subset.";
+
+    return (
+      <Tooltip
+        content={tipContent}
+        position="bottom"
+        hoverOpenDelay={globals.tooltipHoverOpenDelay}
+      >
+        <AnchorButton
+          icon="new-object"
+          style={{ marginRight: 10 }}
+          disabled={disabled}
+          onClick={() => dispatch(actions.requestReembed())}
+          loading={loading}
+        />
+      </Tooltip>
+    );
+  }
+
+  render() {
+    const { layoutChoice } = this.props;
+
+    return (
+      <ButtonGroup
+        style={{
+          marginRight: 10
+        }}
+      >
+        <Popover
+          target={
+            <Tooltip
+              content="Select embedding for visualization"
+              position="bottom"
+              hoverOpenDelay={globals.tooltipHoverOpenDelay}
+            >
+              <Button
+                type="button"
+                data-testid="layout-choice"
+                icon="heatmap"
+                style={{
+                  cursor: "pointer"
+                }}
+              />
+            </Tooltip>
+          }
+          position={Position.BOTTOM_RIGHT}
+          content={
+            <div
+              style={{
+                display: "flex",
+                justifyContent: "flex-start",
+                alignItems: "flex-start",
+                flexDirection: "column",
+                padding: 10
+              }}
+            >
+              <RadioGroup
+                label="Embedding Choice"
+                onChange={this.handleLayoutChoiceChange}
+                selectedValue={layoutChoice.current}
+              >
+                {layoutChoice.available.map(name => (
+                  <Radio label={name} value={name} key={name} />
+                ))}
+              </RadioGroup>
+            </div>
+          }
+        />
+        {this.renderReembedding()}
+      </ButtonGroup>
+    );
+  }
+}
+
+export default Embedding;
@@ -1,33 +1,21 @@
 // jshint esversion: 6
 import React from "react";
 import { connect } from "react-redux";
-import {
-  Button,
-  ButtonGroup,
-  AnchorButton,
-  Tooltip,
-  Popover,
-  Position,
-  RadioGroup,
-  Radio,
-  Icon
-} from "@blueprintjs/core";
-import { World } from "../../util/stateManager";
+import { Button, ButtonGroup, AnchorButton, Tooltip } from "@blueprintjs/core";
+import * as globals from "../../globals";
 import actions from "../../actions";
 import CellSetButton from "./cellSetButtons";
-import InformationMenu from "./infoMenu";
-import UndoRedoReset from "./undoRedoReset";
 import Clip from "./clip";
-import * as globals from "../../globals";
+import Embedding from "./embedding";
+import InformationMenu from "./infoMenu";
+import Subset from "./subset";
+import UndoRedoReset from "./undoRedo";

@connect(state => ({
  universe: state.universe,
  world: state.world,
-  loading: state.controls.loading,
  crossfilter: state.crossfilter,
  differential: state.differential,
-  resettingInterface: state.controls.resettingInterface,
-  layoutChoice: state.layoutChoice,
  graphInteractionMode: state.controls.graphInteractionMode,
  clipPercentileMin: Math.round(100 * (state.world?.clipQuantiles?.min ?? 0)),
  clipPercentileMax: Math.round(100 * (state.world?.clipQuantiles?.max ?? 1)),
@@ -100,61 +88,6 @@ class MenuBar extends React.Component {
    return isDisabled;
  };

-  isResetDisabled = () => {
-    /*
-    Reset should be disabled when all of the following are true:
-      * nothing is selected in the crossfilter
-      * world EQ universe
-      * nothing is colored by
-      * there are no userDefinedGenes or diffexpGenes displayed
-      * scatterplot is not displayed
-      * nothing in cellset1 or cellset2
-      * clip percentiles are [0,100]
-    */
-    const {
-      crossfilter,
-      world,
-      universe,
-      userDefinedGenes,
-      diffexpGenes,
-      colorAccessor,
-      scatterplotXXaccessor,
-      scatterplotYYaccessor,
-      celllist1,
-      celllist2,
-      clipPercentileMin,
-      clipPercentileMax
-    } = this.props;
-
-    if (!crossfilter || !world || !universe) {
-      return false;
-    }
-    const nothingSelected = crossfilter.countSelected() === crossfilter.size();
-    const nothingColoredBy = !colorAccessor;
-    const noGenes = userDefinedGenes.length === 0 && diffexpGenes.length === 0;
-    const scatterNotDpl = !scatterplotXXaccessor || !scatterplotYYaccessor;
-    const nothingInCellsets = !celllist1 && !celllist2;
-
-    return (
-      nothingSelected &&
-      World.worldEqUniverse(world, universe) &&
-      nothingColoredBy &&
-      noGenes &&
-      scatterNotDpl &&
-      nothingInCellsets &&
-      clipPercentileMax === 100 &&
-      clipPercentileMin === 0
-    );
-  };
-
-  resetInterface = () => {
-    const { dispatch } = this.props;
-    dispatch({
-      type: "interface reset started"
-    });
-    dispatch(actions.resetInterface());
-  };
-
  handleClipOnKeyPress = e => {
    /*
    allow only numbers, plus other critical keys which
@@ -229,14 +162,6 @@ class MenuBar extends React.Component {
    this.setState({ pendingClipPercentiles: null });
  };

-  handleLayoutChoiceChange = e => {
-    const { dispatch } = this.props;
-    dispatch({
-      type: "set layout choice",
-      layoutChoice: e.currentTarget.value
-    });
-  };
-
  computeDiffExp = () => {
    const { dispatch, differential } = this.props;
    if (differential.celllist1 && differential.celllist2) {
@@ -269,6 +194,19 @@ class MenuBar extends React.Component {
    });
  };

+  subsetPossible = () => {
+    const { crossfilter } = this.props;
+    return (
+      crossfilter.countSelected() !== 0 &&
+      crossfilter.countSelected() !== crossfilter.size()
+    );
+  };
+
+  subsetResetPossible = () => {
+    const { world, universe } = this.props;
+    return world.nObs !== universe.nObs;
+  };
+
  renderDiffExp() {
    /* diffexp-related buttons may be disabled */
    const { disableDiffexp, differential, diffexpMayBeSlow } = this.props;
@@ -284,8 +222,14 @@ class MenuBar extends React.Component {

    return (
      <ButtonGroup style={{ marginRight: 10 }}>
-        <CellSetButton {...this.props} eitherCellSetOneOrTwo={1} />
-        <CellSetButton {...this.props} eitherCellSetOneOrTwo={2} />
+        <CellSetButton
+          {...this.props} // eslint-disable-line react/jsx-props-no-spreading
+          eitherCellSetOneOrTwo={1}
+        />
+        <CellSetButton
+          {...this.props} // eslint-disable-line react/jsx-props-no-spreading
+          eitherCellSetOneOrTwo={2}
+        />
        {!differential.diffExp ? (
          <Tooltip
            content={tipMessage}
@@ -327,15 +271,12 @@ class MenuBar extends React.Component {
  render() {
    const {
      dispatch,
-      crossfilter,
-      resettingInterface,
      libraryVersions,
      undoDisabled,
      redoDisabled,
      selectionTool,
      clipPercentileMin,
      clipPercentileMax,
-      layoutChoice,
      graphInteractionMode,
      aboutLink,
      showCentroidLabels
@@ -363,29 +304,18 @@ class MenuBar extends React.Component {
        }}
      >
        {this.renderDiffExp()}
-        <Tooltip
-          content="Show only metadata and cells which are currently selected"
-          position="bottom"
-          hoverOpenDelay={globals.tooltipHoverOpenDelay}
-        >
-          <AnchorButton
-            data-testid="subset-button"
-            disabled={
-              crossfilter &&
-              (crossfilter.countSelected() === 0 ||
-                crossfilter.countSelected() === crossfilter.size())
-            }
-            style={{
-              marginRight: 10
-            }}
-            onClick={() => {
-              dispatch(actions.regraph());
-              dispatch({ type: "increment graph render counter" });
-            }}
-          >
-            <Icon icon="double-chevron-down" />
-          </AnchorButton>
-        </Tooltip>
+        <Subset
+          subsetPossible={this.subsetPossible()}
+          subsetResetPossible={this.subsetResetPossible()}
+          handleSubset={() => {
+            dispatch(actions.setWorldToSelection());
+            dispatch({ type: "increment graph render counter" });
+          }}
+          handleSubsetReset={() => {
+            dispatch(actions.resetWorldToUniverse());
+            dispatch({ type: "increment graph render counter" });
+          }}
+        />
        <ButtonGroup style={{ marginRight: "10px" }}>
          <Tooltip
            content={selectionTooltip}
@@ -436,6 +366,8 @@ class MenuBar extends React.Component {
          disabled={graphInteractionMode === "zoom"}
        >
          <Button
+            type="button"
+            data-testid="centroid-label-toggle"
            icon="property"
            onClick={this.handleCentroidChange}
            active={showCentroidLabels}
@@ -445,52 +377,7 @@ class MenuBar extends React.Component {
            }}
          />
        </Tooltip>
-        <ButtonGroup
-          style={{
-            marginRight: 10
-          }}
-        >
-          <Popover
-            target={
-              <Tooltip
-                content="Select embedding for visualization"
-                position="bottom"
-                hoverOpenDelay={globals.tooltipHoverOpenDelay}
-              >
-                <Button
-                  type="button"
-                  data-testid="layout-choice"
-                  icon="heatmap"
-                  style={{
-                    cursor: "pointer"
-                  }}
-                />
-              </Tooltip>
-            }
-            position={Position.BOTTOM_RIGHT}
-            content={
-              <div
-                style={{
-                  display: "flex",
-                  justifyContent: "flex-start",
-                  alignItems: "flex-start",
-                  flexDirection: "column",
-                  padding: 10
-                }}
-              >
-                <RadioGroup
-                  label="Embedding Choice"
-                  onChange={this.handleLayoutChoiceChange}
-                  selectedValue={layoutChoice.current}
-                >
-                  {layoutChoice.available.map(name => (
-                    <Radio label={name} value={name} key={name} />
-                  ))}
-                </RadioGroup>
-              </div>
-            }
-          />
-        </ButtonGroup>
+        <Embedding />
        <Clip
          pendingClipPercentiles={pendingClipPercentiles}
          clipPercentileMin={clipPercentileMin}
@@ -509,9 +396,6 @@ class MenuBar extends React.Component {
        />
        <UndoRedoReset
          dispatch={dispatch}
-          isResetDisabled={this.isResetDisabled}
-          resetInterface={this.resetInterface}
-          resettingInterface={resettingInterface}
          undoDisabled={undoDisabled}
          redoDisabled={redoDisabled}
        />
@@ -0,0 +1,43 @@
+import React from "react";
+import { AnchorButton, ButtonGroup, Tooltip } from "@blueprintjs/core";
+import * as globals from "../../globals";
+
+function Subset(props) {
+  const {
+    subsetPossible,
+    subsetResetPossible,
+    handleSubset,
+    handleSubsetReset
+  } = props;
+
+  return (
+    <ButtonGroup style={{ marginRight: "10px" }}>
+      <Tooltip
+        content="Subset to currently selected cells and associated metadata"
+        position="bottom"
+        hoverOpenDelay={globals.tooltipHoverOpenDelay}
+      >
+        <AnchorButton
+          data-testid="subset-button"
+          disabled={!subsetPossible}
+          icon="pie-chart"
+          onClick={handleSubset}
+        />
+      </Tooltip>
+      <Tooltip
+        content="Undo subset and show all cells and associated metadata"
+        position="bottom"
+        hoverOpenDelay={globals.tooltipHoverOpenDelay}
+      >
+        <AnchorButton
+          data-testid="reset-subset-button"
+          disabled={!subsetResetPossible}
+          icon="full-circle"
+          onClick={handleSubsetReset}
+        />
+      </Tooltip>
+    </ButtonGroup>
+  );
+}
+
+export default Subset;
@@ -5,11 +5,8 @@ import { tooltipHoverOpenDelay } from "../../globals";

 function InformationMenu(props) {
  const {
-    resettingInterface,
    undoDisabled,
    redoDisabled,
-    resetInterface,
-    isResetDisabled,
    dispatch
  } = props;
  return (
@@ -29,6 +26,7 @@ function InformationMenu(props) {
          style={{
            cursor: "pointer"
          }}
+          data-testid="undo"
        />
      </Tooltip>
      <Tooltip
@@ -46,23 +44,7 @@ function InformationMenu(props) {
          style={{
            cursor: "pointer"
          }}
-        />
-      </Tooltip>
-      <Tooltip
-        content="Reset cellxgene, clearing all selections"
-        position="bottom"
-        hoverOpenDelay={tooltipHoverOpenDelay}
-      >
-        <AnchorButton
-          disabled={isResetDisabled()}
-          style={{ marginLeft: 10 }}
-          type="button"
-          loading={resettingInterface}
-          intent="none"
-          icon="refresh"
-          onClick={resetInterface}
-          data-testid="reset"
-          data-testclass={`resetting-${resettingInterface}`}
+          data-testid="redo"
        />
      </Tooltip>
    </div>
@@ -1,7 +1,8 @@
 import { Colors } from "@blueprintjs/core";
+import { dispatchNetworkErrorMessageToUser } from "./util/actionHelpers";

 /* if a categorical metadata field has more options than this, truncate */
-export const maxCategoricalOptionsToDisplay = 100;
+export const maxCategoricalOptionsToDisplay = 200;

 /* default "unassigned" value for user-created categorical metadata */
 export const unassignedCategoryLabel = "unassigned";
@@ -72,18 +73,32 @@ export const leftSidebarSectionHeading = {
 export const leftSidebarSectionPadding = 10;
 export const categoryLabelDisplayStringLongLength = 27;
 export const categoryLabelDisplayStringShortLength = 11;
+export const categoryDisplayStringMaxLength = 27;
+
+export const maxUserDefinedGenes = 25;
+export const maxGenes = 100;

 /* various timing-related behaviors */
 export const tooltipHoverOpenDelay = 1000; /* ms delay before a tooltip displays */
 export const tooltipHoverOpenDelayQuick = 500;

-let _API = {
-  // prefix: "http://api.clustering.czi.technology/api/",
-  // prefix: "http://tabulamuris.cxg.czi.technology/api/",
-  // prefix: "http://api-staging.clustering.czi.technology/api/",
-  prefix: "http://localhost:5005/api/",
-  version: "v0.2/"
-};
+let _API;
+
+if (window.CELLXGENE && window.CELLXGENE.API) {
+  _API = window.CELLXGENE.API;
+} else {
+  if (process.env.CXG_SERVER_PORT === undefined) {
+    const errorMessage = "Please set the CXG_SERVER_PORT environment variable.";
+    dispatchNetworkErrorMessageToUser(errorMessage);
+    throw new Error(errorMessage);
+  }
+  _API = {
+    // prefix: "http://api.clustering.czi.technology/api/",
+    // prefix: "http://tabulamuris.cxg.czi.technology/api/",
+    // prefix: "http://api-staging.clustering.czi.technology/api/",
+    prefix: `http://localhost:${process.env.CXG_SERVER_PORT}/api/`,
+    version: "v0.2/"
+  };
+}

-if (window.CELLXGENE && window.CELLXGENE.API) _API = window.CELLXGENE.API;
 export const API = _API;
@@ -55,61 +55,40 @@ const Annotations = (
    }

    /* CATEGORY */
-    case "annotation: activate add new label mode":
+    case "annotation: activate add new label mode": {
      return {
        ...state,
        isAddingNewLabel: true,
        categoryAddingNewLabel: action.data
      };
-    case "annotation: activate add new ontology label mode":
-      return {
-        ...state,
-        isAddingNewLabelFromOntology: true,
-        categoryAddingNewLabelFromOntology: action.data
-      };
-    case "annotation: disable add new ontology label mode":
-      return {
-        ...state,
-        isAddingNewLabelFromOntology: false,
-        categoryAddingNewLabelFromOntology: null
-      };
+    }

-    case "annotation: disable add new label mode":
+    case "annotation: disable add new label mode": {
      return {
        ...state,
        isAddingNewLabel: false,
        categoryAddingNewLabel: null
      };
-    /* this handles both cases, since at this point it's just a string */
-    case "annotation: add new label to category":
-      return {
-        ...state,
-        isAddingNewLabel: false,
-        categoryAddingNewLabel: null,
-        isAddingNewLabelFromOntology: false,
-        categoryAddingNewLabelFromOntology: null
-      };
-    case "annotation: activate category edit mode":
+    }
+
+    case "annotation: activate category edit mode": {
      return {
        ...state,
        isEditingCategoryName: true,
        categoryBeingEdited: action.data
      };
-    case "annotation: disable category edit mode":
-      return {
-        ...state,
-        isEditingCategoryName: false,
-        categoryBeingEdited: null
-      };
-    case "annotation: category edited":
+    }
+
+    case "annotation: disable category edit mode": {
      return {
        ...state,
        isEditingCategoryName: false,
        categoryBeingEdited: null
      };
+    }

    /* LABEL */
-    case "annotation: activate edit label mode":
+    case "annotation: activate edit label mode": {
      return {
        ...state,
        isEditingLabelName: true,
@@ -118,18 +97,16 @@ const Annotations = (
          label: action.categoryIndex
        }
      };
-    case "annotation: cancel edit label mode":
-      return {
-        ...state,
-        isEditingLabelName: false,
-        labelEditable: { category: null, label: null }
-      };
-    case "annotation: label edited":
+    }
+
+    case "annotation: cancel edit label mode": {
      return {
        ...state,
        isEditingLabelName: false,
        labelEditable: { category: null, label: null }
      };
+    }
+
    default:
      return state;
  }
@@ -2,7 +2,8 @@ const Autosave = (
 	state = {
 		saveInProgress: false,
 		error: false,
-		lastSavedObsAnnotations: null
+		lastSavedObsAnnotations: null,
+		initialDataLoadComplete: false
 	},
 	action,
 	nextSharedState
@@ -15,7 +16,8 @@ const Autosave = (
 				...state,
 				error: false,
 				saveInProgress: false,
-				lastSavedObsAnnotations: universe.obsAnnotations
+				lastSavedObsAnnotations: universe.obsAnnotations,
+				initialDataLoadComplete: true,
 			};
 		}

@@ -1,6 +1,6 @@
 export default function cascadeReducers(arg) {
  /*
-  Combined a set of cascading reducers into a single reducer.  Cascading
+  Combine a set of cascading reducers into a single reducer.  Cascading
  reducers are reducers which may rely on state computed by another reducer.
  Therefore, they:
  - must be composed in a particular order (currently, this is a simple
@@ -13,7 +13,7 @@ export default function cascadeReducers(arg) {
  - an array of tuples, [ [key1, reducer1], [key2, reducer2], ... ]
  Ie,  cascadeReducers([ ["a", reduceA], ["b", reduceB] ])

-  Each reducer will be called with the sigature:
+  Each reducer will be called with the signature:
      (prevState, action, sharedNextState, sharedPrevState) => newState

  cascadeReducers will build a composite newState object, much
@@ -1,12 +1,4 @@
 import { ControlsHelpers as CH } from "../util/stateManager";
-import * as globals from "../globals";
-
-function maxCategoryItems(state) {
-  return (
-    state.config.parameters?.["max-category-items"] ??
-    globals.configDefaults.parameters["max-category-items"]
-  );
-}

 const CategoricalSelection = (
  state,
@@ -22,11 +14,32 @@ const CategoricalSelection = (
      const { world } = nextSharedState;
      const newState = CH.createCategoricalSelection(
        world,
-        CH.selectableCategoryNames(world, maxCategoryItems(prevSharedState))
+        CH.selectableCategoryNames(
+          world.schema,
+          CH.maxCategoryItems(prevSharedState.config)
+        )
      );
      return newState;
    }

+    case "universe: column load success": {
+      const { dim } = action;
+      if (dim !== "obsAnnotations") return state;
+
+      const { dataframe } = action;
+      const { world } = nextSharedState;
+      const names = CH.selectableCategoryNames(
+        world.schema,
+        CH.maxCategoryItems(prevSharedState.config),
+        dataframe.colIndex.keys()
+      );
+      if (names.length === 0) return state;
+      return {
+        ...state,
+        ...CH.createCategoricalSelection(world, names)
+      };
+    }
+
    case "categorical metadata filter select": {
      /*
      Set the specific category in this field to false
@@ -12,12 +12,12 @@ const ColorsReducer = (
  prevSharedState
 ) => {
  switch (action.type) {
-    case "initial data load complete (universe exists)":
-    case "reset World to eq Universe": {
+    case "universe exists, but loading is still in progress": {
+      /* initialize everything with default colors, no mode, no color-by accessor */
      const { world } = nextSharedState;
      const colorMode = null;
      const colorAccessor = null;
-      const { rgb, scale } = ColorHelpers.createColors(world, colorMode);
+      const { rgb, scale } = ColorHelpers.createColors(world);
      return {
        ...state,
        colorAccessor,
@@ -27,6 +27,22 @@ const ColorsReducer = (
      };
    }

+    case "reset World to eq Universe": {
+      /* need to rebuild colors as world may have changed, but don't switch modes */
+      const { world } = nextSharedState;
+      const { colorMode, colorAccessor } = state;
+      const { rgb, scale } = ColorHelpers.createColors(
+        world,
+        colorMode,
+        colorAccessor
+      );
+      return {
+        ...state,
+        rgb,
+        scale
+      };
+    }
+
    case "set clip quantiles":
    case "set World to current selection": {
      const { world: prevWorld, controls: prevControls } = prevSharedState;
@@ -1,6 +1,7 @@
 // jshint esversion: 6

 import _ from "lodash";
+import { subsetAndResetGeneLists } from "../util/stateManager/controlsHelpers";

 const Controls = (
  state = {
@@ -41,8 +42,31 @@ const Controls = (
    case "initial data load start": {
      return { ...state, loading: true };
    }
+    case "universe: column load success": {
+      /* 
+      we are in a loading state until the following are true:
+        * universe exists (if partially)
+        * embeddings (obsLayout) are loaded
+        * varAnnotations index is loaded
+      */
+      const universeExists = !!nextSharedState.universe;
+      const embeddingsExist =
+        !nextSharedState.universe?.obsLayout?.isEmpty() ?? false;
+      const varIndex = nextSharedState.universe.schema.annotations.var.index;
+      const varAnnotationsIndexExists =
+        universeExists &&
+        nextSharedState.universe.varAnnotations.hasCol(varIndex);
+      return {
+        ...state,
+        loading: !(
+          universeExists &&
+          embeddingsExist &&
+          varAnnotationsIndexExists
+        )
+      };
+    }
    case "initial data load complete (universe exists)": {
-      /* first light - create world & other data-driven defaults */
+      /* now fully loaded */
      return {
        ...state,
        loading: false,
@@ -51,16 +75,22 @@ const Controls = (
      };
    }
    case "reset World to eq Universe": {
+      const [ newUserDefinedGenes, newDiffExpGenes ] = subsetAndResetGeneLists(state);
      return {
        ...state,
-        resettingInterface: false
+        resettingInterface: false,
+        userDefinedGenes: newUserDefinedGenes,
+        diffexpGenes: newDiffExpGenes
      };
    }
    case "set World to current selection": {
+      const [ newUserDefinedGenes, newDiffExpGenes ] = subsetAndResetGeneLists(state);
      return {
        ...state,
        loading: false,
-        error: null
+        error: null,
+        userDefinedGenes: newUserDefinedGenes,
+        diffexpGenes: newDiffExpGenes
      };
    }
    case "request user defined gene started": {
@@ -23,13 +23,37 @@ const CrossfilterReducerBase = (
  prevSharedState
 ) => {
  switch (action.type) {
-    case "initial data load complete (universe exists)": {
+    case "universe: column load success": {
      const { world, layoutChoice } = nextSharedState;
-      const crossfilter = World.createObsDimensions(
-        new Crossfilter(world.obsAnnotations),
-        world,
-        layoutChoice.currentDimNames
-      );
+      const { obsAnnotations, obsLayout } = world;
+
+      // ignore var dimension loads as these are not currently selectable
+      if (action.dim === "varAnnotations") return state;
+
+      /* 
+      during bootstrap loading, we don't know if obsLayout or obsAnnotations
+      will load first. Take whichever arrives and is not empty (so that our
+      crossfilter has the right dimensionality).
+      */
+      let crossfilter =
+        state ??
+        new Crossfilter(obsAnnotations.isEmpty() ? obsLayout : obsAnnotations);
+
+      // add layout dimension, if not already present
+      if (
+        obsLayout.hasCol(layoutChoice.currentDimNames[0]) &&
+        !crossfilter.hasDimension(XYDimName)
+      ) {
+        crossfilter = crossfilter.addDimension(
+          XYDimName,
+          "spatial",
+          obsLayout.col(layoutChoice.currentDimNames[0]).asArray(),
+          obsLayout.col(layoutChoice.currentDimNames[1]).asArray()
+        );
+      }
+
+      // add any missing obsAnnotations
+      crossfilter = World.addObsDimensions(crossfilter, world);
      return crossfilter;
    }

@@ -278,10 +302,19 @@ const CrossfilterReducer = (
    nextSharedState,
    prevSharedState
  );
-  if (!nextState || nextState.all() === nextSharedState.world.obsAnnotations) {
+  /*
+  update the data in the crossfilter to point at the current obsAnnotations, IF
+  they are not empty.  If empty, leave it alone (can occur during boostrap loading).
+  */
+  const nextObsAnnotations = nextSharedState.world?.obsAnnotations;
+  if (
+    !nextState ||
+    nextState.all() === nextObsAnnotations ||
+    nextObsAnnotations.isEmpty()
+  ) {
    return nextState;
  }
-  return nextState.setData(nextSharedState.world.obsAnnotations);
+  return nextState.setData(nextObsAnnotations);
 };

 export default CrossfilterReducer;
@@ -21,6 +21,7 @@ import autosave from "./autosave";
 import ontology from "./ontology";
 import centroidLabels from "./centroidLabels";
 import pointDialation from "./pointDilation";
+import { reembedController, reembedding } from "./reembed";

 import undoableConfig from "./undoableConfig";

@@ -31,6 +32,7 @@ const Reducer = undoable(
    ["world", world],
    ["ontology", ontology],
    ["annotations", annotations],
+    ["reembedding", reembedding],
    ["layoutChoice", layoutChoice],
    ["categoricalSelection", categoricalSelection],
    ["continuousSelection", continuousSelection],
@@ -42,6 +44,7 @@ const Reducer = undoable(
    ["responsive", responsive],
    ["centroidLabels", centroidLabels],
    ["pointDilation", pointDialation],
+    ["reembedController", reembedController],
    ["autosave", autosave],
    ["resetCache", resetCache]
  ]),
@@ -57,7 +60,8 @@ const Reducer = undoable(
    "differential",
    "layoutChoice",
    "centroidLabels",
-    "annotations"
+    "annotations",
+    "reembedding"
  ],
  undoableConfig
 );
@@ -14,6 +14,14 @@ function bestDefaultLayout(layouts) {
  return layouts[0];
 }

+function setToDefaultLayout(world) {
+  const { schema } = world;
+  const available = schema.layout.obs.map(v => v.name).sort();
+  const current = bestDefaultLayout(available);
+  const currentDimNames = schema.layout.obsByName[current].dims;
+  return { available, current, currentDimNames };
+}
+
 const LayoutChoice = (
  state = {
    available: [], // all available choices
@@ -24,14 +32,13 @@ const LayoutChoice = (
  nextSharedState
 ) => {
  switch (action.type) {
-    case "initial data load complete (universe exists)":
-    case "reset World to eq Universe": {
+    case "universe exists, but loading is still in progress": {
      // set default to default
-      const { schema } = nextSharedState.world;
-      const available = schema.layout.obs.map(v => v.name).sort();
-      const current = bestDefaultLayout(available);
-      const currentDimNames = schema.layout.obsByName[current].dims;
-      return { available, current, currentDimNames };
+      const { universe } = nextSharedState;
+      return {
+        ...state,
+        ...setToDefaultLayout(universe)
+      };
    }

    case "set layout choice": {
@@ -41,6 +48,31 @@ const LayoutChoice = (
      return { ...state, current, currentDimNames };
    }

+    case "reembed: add reembedding": {
+      const name = action.schema.name;
+      const available = Array.from(new Set(state.available).add(name));
+      return {
+        ...state,
+        available
+      };
+    }
+
+    case "reembed: clear all reembeddings": {
+      const { universe } = nextSharedState;
+      const { current } = state;
+      const dflt = setToDefaultLayout(universe);
+      if (dflt.available.includes(current)) {
+        return {
+          ...state,
+          available: dflt.available
+        };
+      }
+      return {
+        ...state,
+        ...dflt
+      };
+    }
+
    default: {
      return state;
    }
@@ -1,4 +1,3 @@
-// jshint esversion: 6
 const Ontology = (
  state = {
    enabled: false, // are ontology terms enabled?
@@ -9,10 +8,12 @@ const Ontology = (
  action
 ) => {
  switch (action.type) {
-    case "configuration load complete":
+    case "configuration load complete": {
+      /* eslint-disable camelcase */
      const enabled =
        action.config?.parameters?.annotations_cell_ontology_enabled ?? false;
      const terms = action.config?.parameters?.annotations_cell_ontology_terms;
+      /* eslint-enable camelcase */
      const termSet = new Set(terms);
      return {
        ...state,
@@ -21,8 +22,10 @@ const Ontology = (
        terms,
        termSet
      };
-    default:
+    }
+    default: {
      return state;
+    }
  }
 };

@@ -0,0 +1,64 @@
+/*
+controller state is not part of the undo/redo history
+*/
+export const reembedController = (
+  state = {
+    pendingFetch: null
+  },
+  action
+) => {
+  switch (action.type) {
+    case "reembed: request start": {
+      return {
+        ...state,
+        pendingFetch: action.abortableFetch
+      };
+    }
+    case "reembed: request aborted":
+    case "reembed: request cancel":
+    case "reembed: request completed": {
+      return {
+        ...state,
+        pendingFetch: null
+      };
+    }
+    default: {
+      return state;
+    }
+  }
+};
+
+/*
+actual reembedding data is part of the undo/redo history
+*/
+export const reembedding = (
+  state = {
+    reembeddings: new Map()
+  },
+  action
+) => {
+  switch (action.type) {
+    case "reembed: add reembedding": {
+      const { schema, embedding } = action;
+      const { name } = schema.name;
+      const { reembeddings } = state;
+      return {
+        ...state,
+        reembeddings: new Map(reembeddings).set(name, {
+          name,
+          schema,
+          embedding
+        })
+      };
+    }
+    case "reembed: clear all reembeddings": {
+      return {
+        ...state,
+        reembeddings: new Map()
+      };
+    }
+    default: {
+      return state;
+    }
+  }
+};
@@ -85,7 +85,6 @@ const Undoable = (reducer, undoableKeys, options = {}) => {
    );
    const newPast = [...past];
    const newState = newPast.pop();
-    const newStateFilterState = newState[filterStateKey];
    const newFuture = push(future, currentUndoableState);
    const nextState = {
      ...currentState,
@@ -94,7 +93,6 @@ const Undoable = (reducer, undoableKeys, options = {}) => {
      [futureKey]: newFuture,
      [pendingKey]: null
    };
-    nextState[filterStateKey] = newStateFilterState;
    return nextState;
  }

@@ -139,12 +137,13 @@ const Undoable = (reducer, undoableKeys, options = {}) => {
  */
  function skip(currentState, action, filterState) {
    const past = currentState[pastKey] || [];
+    const future = currentState[futureKey] || [];
    const pending = currentState[pendingKey];
    const res = reducer(currentState, action);
    return {
      ...res,
      [pastKey]: past,
-      [futureKey]: [],
+      [futureKey]: future,
      [filterStateKey]: filterState,
      [pendingKey]: pending
    };
@@ -218,18 +217,19 @@ const Undoable = (reducer, undoableKeys, options = {}) => {
    },
    action
  ) => {
+    if (debug > 1) console.log("---- ACTION", action.type);
    const aType = action.type;
    switch (aType) {
      case "@@undoable/undo": {
-        return undo(currentState, action);
+        return undo(currentState);
      }

      case "@@undoable/redo": {
-        return redo(currentState, action);
+        return redo(currentState);
      }

      case "@@undoable/clear": {
-        return clear(currentState, action);
+        return clear(currentState);
      }

      default: {
@@ -11,10 +11,11 @@ const skipOnActions = new Set([
  "url changed",
  "interface reset started",
  "initial data load start",
+  "universe: column load success",
+  "universe exists, but loading is still in progress",
  "configuration load complete",
  "increment graph render counter",
  "window resize",
-  "user reset start",
  "reset colorscale",
  "reset centroid labels",

@@ -33,18 +34,28 @@ const skipOnActions = new Set([
  "get single gene expression for coloring error",

  "category value mouse hover start",
-  "category value mouse hover end"
+  "category value mouse hover end",
+
+  /* autosave annotations */
+  "writable obs annotations - save complete",
+  "writable obs annotations - save started",
+  "writable obs annotations - save error",
+
+  /* annotation component action */
+  "annotation: activate add new label mode",
+  "annotation: disable add new label mode",
+  "annotation: activate category edit mode",
+  "annotation: disable category edit mode",
+  "annotation: activate edit label mode",
+  "annotation: cancel edit label mode",
+  "set annotations collection name"
 ]);

 /*
 identical, repeated occurances of these action types will be debounced.
 Entire action must be identical (all keys).
 */
-const debounceOnActions = new Set([
-  "color by categorical metadata",
-  "color by continuous metadata",
-  "color by expression"
-]);
+const debounceOnActions = new Set([]);

 /*
 history will be cleared when these actions occur
@@ -53,7 +64,6 @@ const clearOnActions = new Set([
  "initial data load complete (universe exists)",
  "reset World to eq Universe",
  "initial data load error",
-  "user reset end"
 ]);

 /*
@@ -89,7 +99,8 @@ const saveOnActions = new Set([
  "annotation: delete category",
  "annotation: label edited",
  "annotation: label current cell selection",
-  "annotation: delete label"
+  "annotation: delete label",
+  "annotation: category edited"
 ]);

 /**
@@ -134,7 +145,7 @@ Signature:  (fsm, event, from) => undoableAction
 const onFsmError = (fsm, event, from) => {
  console.error(`FSM error [event: "${event}", state: "${from}"]`, fsm);
  // In production, try to recover gracefully if we have unexpected state
-  return clear(fsm);
+  return clear();
 };

 /*
@@ -170,8 +181,6 @@ const actionFilter = debug => (state, action, prevFilterState) => {
  }
  if (
    debounceOnActions.has(actionType) &&
-    prevFilterState !== undefined &&
-    prevFilterState.prevAction !== undefined &&
    shallowObjectEq(action, prevFilterState.prevAction)
  ) {
    return { [actionKey]: "skip", [stateKey]: filterState };
@@ -227,7 +236,12 @@ function shallowArrayEq(arrA, arrB) {
 }

 /* configuration for the undoable meta reducer */
-const debug = false; // set truish for undoble debugging
+/*
+debug: set to any falsish value to disable logging of helpful debugging information.
+Set to true or 1 for base logging, high number for more verbosity (currently only 1/true
+or 2).
+*/
+const debug = false;
 const undoableConfig = {
  debug,
  historyLimit: 50, // maximum history size
@@ -1,4 +1,9 @@
 import { unassignedCategoryLabel } from "../globals";
+import {
+  addObsAnnotations,
+  addVarAnnotations,
+  addObsLayout
+} from "../util/stateManager/universe";
 import {
  World,
  ControlsHelpers,
@@ -7,11 +12,38 @@ import {

 const Universe = (state = null, action, nextSharedState, prevSharedState) => {
  switch (action.type) {
-    case "initial data load complete (universe exists)": {
+    case "universe exists, but loading is still in progress": {
      const { universe } = action;
      return universe;
    }

+    case "universe: column load success": {
+      const { dim, dataframe } = action;
+      switch (dim) {
+        case "obsAnnotations": {
+          return {
+            ...state,
+            ...addObsAnnotations(state, dataframe)
+          };
+        }
+        case "varAnnotations": {
+          return {
+            ...state,
+            ...addVarAnnotations(state, dataframe)
+          };
+        }
+        case "obsLayout": {
+          return {
+            ...state,
+            ...addObsLayout(state, dataframe)
+          };
+        }
+        default: {
+          throw new Error("action handler not implemented");
+        }
+      }
+    }
+
    case "expression load success": {
      let { varData } = state;

@@ -30,9 +62,7 @@ const Universe = (state = null, action, nextSharedState, prevSharedState) => {
      const { userDefinedGenes, diffexpGenes } = prevSharedState;
      const allTheGenesWeNeed = [
        ...new Set(
-          userDefinedGenes,
-          diffexpGenes,
-          Object.keys(action.expressionData)
+          [userDefinedGenes, diffexpGenes, Object.keys(action.expressionData)].filter(ele => ele).flat()
        )
      ];
      varData = ControlsHelpers.pruneVarDataCache(varData, allTheGenesWeNeed);
@@ -4,6 +4,10 @@ import {
  ControlsHelpers,
  AnnotationsHelpers
 } from "../util/stateManager";
+import {
+  addObsLayout,
+  removeObsLayout
+} from "../util/stateManager/schemaHelpers";
 import clip from "../util/clip";
 import quantile from "../util/quantile";

@@ -24,13 +28,28 @@ const WorldReducer = (
  prevSharedState
 ) => {
  switch (action.type) {
-    case "initial data load complete (universe exists)":
+    case "universe exists, but loading is still in progress":
    case "reset World to eq Universe": {
      const { universe } = nextSharedState;
      const world = World.createWorldFromEntireUniverse(universe);
      return world;
    }

+    case "universe: column load success": {
+      /* incremental initial data load - always assumes world == universe */
+      const { universe } = nextSharedState;
+      const { dim } = action;
+      return {
+        ...state,
+        schema: universe.schema,
+        [dim]: universe[dim].clone(),
+        unclipped: {
+          ...state.unclipped,
+          [dim]: universe[dim].clone()
+        }
+      };
+    }
+
    case "set World to current selection": {
      /* Set viewable world to be the currently selected data */
      const world = World.createWorldBySelection(
@@ -87,15 +106,10 @@ const WorldReducer = (
      const { userDefinedGenes, diffexpGenes } = prevSharedState;
      const allTheGenesWeNeed = [
        ...new Set(
-          userDefinedGenes,
-          diffexpGenes,
-          Object.keys(action.expressionData)
+          [userDefinedGenes, diffexpGenes, Object.keys(action.expressionData)].filter(ele => ele).flat()
        )
      ];
-      unclippedVarData = ControlsHelpers.pruneVarDataCache(
-        unclippedVarData,
-        allTheGenesWeNeed
-      );
+      unclippedVarData = ControlsHelpers.pruneVarDataCache(unclippedVarData, allTheGenesWeNeed);

      // at this point, we have the unclipped data in unclippedVarData.
      // Now create clipped.
@@ -267,6 +281,52 @@ const WorldReducer = (
      };
    }

+    case "reembed: add reembedding": {
+      // new embedding loaded, which *only* affects world's layout.
+      // It may be new, or it may replace a previous re-embedding.
+      const { obsLayout: origObsLayout, schema: origSchema } = state;
+      const { embedding, schema: embeddingSchema } = action;
+
+      const { dims, name } = embeddingSchema;
+      let obsLayout = origObsLayout;
+      let schema = origSchema;
+
+      // alias the names the server sent us, in case they were not the same as the schema
+      const embedingLabels = embedding.colIndex.keys();
+      const labels = {
+        [embedingLabels[0]]: dims[0],
+        [embedingLabels[1]]: dims[1]
+      };
+      obsLayout = obsLayout.withColsFrom(embedding, labels);
+      schema = addObsLayout(schema, embeddingSchema);
+      return {
+        ...state,
+        obsLayout,
+        schema
+      };
+    }
+
+    case "reembed: clear all reembeddings": {
+      // reembedding was cleared -- remove from layout
+      const { obsLayout: origObsLayout, schema: origSchema } = state;
+      const { reembedding } = prevSharedState;
+
+      let schema = origSchema;
+      let obsLayout = origObsLayout;
+
+      reembedding.reembeddings.forEach((emb, name) => {
+        const { dims } = emb.schema;
+        obsLayout = obsLayout.dropCol(dims[0]);
+        obsLayout = obsLayout.dropCol(dims[1]);
+        schema = removeObsLayout(schema, name);
+      });
+      return {
+        ...state,
+        obsLayout,
+        schema
+      };
+    }
+
    default: {
      return state;
    }
@@ -10,7 +10,6 @@ TL;DR: sort order is:

 import isNumber from "is-number";
 import * as globals from "../globals";
-import { memoize } from "./dataframe/util";

 function caseInsensitiveCompare(a, b) {
  const textA = String(a).toUpperCase();
@@ -4,13 +4,96 @@ import { unassignedCategoryLabel } from "../globals";

 /*
  Centroid coordinate calculation
+  Calculates centroids for displaying
+  In the case that a category is truncated, the truncated labels will not have
+    centroids calculated
 */

-/* 
-  calcMedianCentroid goes through a given metadata category
-  fetches each cell's coordinates grouping by category value.
+/*
+Generates a mapping of categorical values to data needed to calculate centroids
+categoricalValue -> {
+ length: int,
+ holdsFinite: Boolean,
+ xCoordinates: Float32Array,
+ yCoordinates: Float32Array
+}
+*/
+const getCoordinatesByCategoricalValues = (
+  obsAnnotations,
+  obsLayout,
+  categoryName,
+  layoutDimNames,
+  categoricalSelection,
+  schemaObsByName
+) => {
+  const coordsByCategoryLabel = new Map();

-  It then calculates the median value and puts that in the array
+  const categoryArray = obsAnnotations.col(categoryName).asArray();
+
+  const layoutXArray = obsLayout.col(layoutDimNames[0]).asArray();
+  const layoutYArray = obsLayout.col(layoutDimNames[1]).asArray();
+
+  const { categoryValueIndices, categoryValueCounts } =
+    categoricalSelection?.[categoryName] || {};
+
+  // If the coloredBy is not a categorical col
+  if (categoryValueIndices === undefined) {
+    return coordsByCategoryLabel;
+  }
+
+  // Check to see if the current category is a user created annotation
+  const isUserAnno = schemaObsByName[categoryName].writable;
+
+  // Iterate over all cells
+  for (let i = 0, len = categoryArray.length; i < len; i += 1) {
+    // Fetch the categorical value of the current cell
+    const categoryValue = categoryArray[i];
+
+    // Get the index of the categoryValue within the category
+    const categoryValueIndex = categoryValueIndices.get(categoryValue);
+
+    // If the category is truncated and this value is removed,
+    //  it will not be assigned a category value and will not be
+    //  labeled on the graph
+    // If the user created this category,
+    //  do not create a label for the `unassigned` value
+    if (
+      categoryValueIndex !== undefined &&
+      !(isUserAnno && categoryValue === unassignedCategoryLabel)
+    ) {
+      // Create/fetch the scratchpad value
+      let coords = coordsByCategoryLabel.get(categoryValue);
+      if (coords === undefined) {
+        // Get the number of cells which are in the categorical value
+        const numInCategoricalValue = categoryValueCounts[categoryValueIndex];
+        coords = {
+          hasFinite: false,
+          xCoordinates: new Float32Array(numInCategoricalValue),
+          yCoordinates: new Float32Array(numInCategoricalValue),
+          length: 0
+        };
+        coordsByCategoryLabel.set(categoryValue, coords);
+      }
+
+      coords.hasFinite =
+        coords.hasFinite ||
+        (Number.isFinite(layoutXArray[i]) && Number.isFinite(layoutYArray[i]));
+
+      const coordinatesLength = coords.length;
+
+      coords.xCoordinates[coordinatesLength] = layoutXArray[i];
+      coords.yCoordinates[coordinatesLength] = layoutYArray[i];
+
+      coords.length = coordinatesLength + 1;
+    }
+  }
+  return coordsByCategoryLabel;
+};
+
+/* 
+  calcMedianCentroid calculates the median coordinates for categorical values in a given metadata field 
+
+  categoricalValue -> [x-Coordinate, y-Coordinate]
 */

 const calcMedianCentroid = (
@@ -21,85 +104,39 @@ const calcMedianCentroid = (
  categoricalSelection,
  schemaObsByName
 ) => {
-  const categoryArray = obsAnnotations.col(categoryName).asArray();
+  // generate a map describing the coordinates for each value within the given category
+  const dataMap = getCoordinatesByCategoricalValues(
+    obsAnnotations,
+    obsLayout,
+    categoryName,
+    layoutDimNames,
+    categoricalSelection,
+    schemaObsByName
+  );

-  const layoutXArray = obsLayout.col(layoutDimNames[0]).asArray();
-  const layoutYArray = obsLayout.col(layoutDimNames[1]).asArray();
+  // categoricalValue => [medianXCoordinate, medianYCoordinate]
  const coordinates = new Map();

-  // Iterate over all the cells in the category
-  for (let i = 0, len = categoryArray.length; i < len; i += 1) {
-    const categoryValue = categoryArray[i];
-
-    // Get the index of the categoryValue within the category
-    // If the category is truncated and this value is removed,
-    // it will not be assigned a category value and will not be
-    // labeled on the graph
-    const categoryValueIndex = categoricalSelection[
-      categoryName
-    ].categoryValueIndices.get(categoryValue);
-
-    // Check to see if the current category is a user created annotation
-    // if the user created this category, do not create a label for the `unassigned` value
-    const isUserAnno = schemaObsByName[categoryName].writable;
-
-    if (
-      categoryValueIndex !== undefined &&
-      !(isUserAnno && categoryValue === unassignedCategoryLabel)
-    ) {
-      // Get the number of cells which are in the category value
-      const numInCategoryValue =
-        categoricalSelection[categoryName].categoryValueCounts[
-          categoryValueIndex
-        ];
-
-      // Create/fetch the valueArray,
-      // which is what the key points to in the `coordinates` hashmap
-      const valueArray = coordinates.get(categoryValue) || [
-        false, // hasFinite
-        0, // index
-        new Float32Array(numInCategoryValue), // x coordinates
-        new Float32Array(numInCategoryValue) // y coordinates
-      ];
-      const index = valueArray[1];
-      let hasFinite = valueArray[0];
-
-      hasFinite =
-        hasFinite ||
-        (Number.isFinite(layoutXArray[i]) && Number.isFinite(layoutYArray[i]));
-
-      valueArray[0] = hasFinite;
-      valueArray[1] = index + 1;
-      valueArray[2][index] = layoutXArray[i];
-      valueArray[3][index] = layoutYArray[i];
-
-      coordinates.set(categoryValue, valueArray);
-    }
-  }
-
  // Iterate over the recently created map
-  coordinates.forEach((value, key) => {
-    // If there are coordinates for this cateogrical value,
+  dataMap.forEach((value, key) => {
+    // If there are coordinates for this categorical value,
    // and there is a finite coordinate for the category value
-    if (value[2].length > 0 && value[3].length > 0 && value[0]) {
-      // Find the median x and y coordinate
-      // and insert them into the first two indices
-      value[0] = quantile([0.5], value[2])[0];
-      value[1] = quantile([0.5], value[3])[0];
-      // Remove the last two elements (where the arrays of coordinates were)
-      value.pop();
-      value.pop();
-    } else {
-      // remove the entry if not
-      coordinates.delete(key);
+    if (value.length > 0 && value.hasFinite) {
+      const calculatedCoordinates = [];
+
+      // Find and store the median x and y coordinate
+      calculatedCoordinates[0] = quantile([0.5], value.xCoordinates)[0];
+      calculatedCoordinates[1] = quantile([0.5], value.yCoordinates)[0];
+
+      coordinates.set(key, calculatedCoordinates);
    }
  });
+
  // return the map: categoricalValue -> [medianXCoordinate, medianYCoordinate]
  return coordinates;
 };

 // A simple function to hash the parameters
-// (not 100% on world hash, Bruce will have to check this one out)
 const hashMedianCentroid = (
  obsAnnotations,
  obsLayout,
@@ -114,5 +151,5 @@ const hashMedianCentroid = (
    categorySelection
  )}:${Object.keys(schemaObsByName)}`;
 };
-// export the mmemoized calculation function
+// export the memoized calculation function
 export default memoize(calcMedianCentroid, hashMedianCentroid);
@@ -348,22 +348,90 @@ class Dataframe {
    );
  }

-  withColsFrom(dataframe) {
+  withColsFrom(dataframe, labels) {
    /*
    return a new dataframe containing all columns from both `this` and the
-    provided dataframe.
+    provided dataframe argument.

-    The row index from `this` will be used.  Both dataframes must have identical
+    The row index from `this` will be used.  All dataframes must have identical
    dimensionality, and no overlapping columns labels.
+
+    Special case, if either dataframe is empty, the other is returned unchanged.
+
+    Arguments:
+    * dataframe: a dataframe to combine with `this`
+    * labels: columns to pull from `dataframe` and combine with `this`.  If falsey,
+      all columns are used.  If an array, must contain a list of labels.  If an
+      Object or Map, the key is the columns to pull, which will be stored into the
+      new dataframe as the value.
+
+    Example:
+
+    newDf = df.withColsFrom(otherDf);         // combines all columns from both
+    newDf = df.withColsFrom(otherDf, ['a']);  // combines df with otherDf['a']
+    newDf = df.withColsFrom(otherDf, {a: 'b'}); // combines df with otherDf['a'], but calls it 'b'
+
     */
-    const dims = [this.dims[0], this.dims[1] + dataframe.dims[1]];
+
+    // resolve the source and dest label names.
+    let srcLabels;
+    let dstLabels;
+    if (!labels) {
+      // combine all columns
+      dstLabels = dataframe.colIndex.keys();
+      srcLabels = dstLabels;
+    } else if (Array.isArray(labels)) {
+      // combine subset of keys with no aliasing
+      dstLabels = labels;
+      srcLabels = labels;
+    } else if (labels instanceof Map) {
+      // aliasing with a Map
+      srcLabels = Array.from(labels.keys());
+      dstLabels = Array.from(labels.values());
+    } else {
+      // aliasing with an Object
+      srcLabels = Object.keys(labels);
+      dstLabels = Object.values(labels);
+    }
+
+    // if datafame is empty, and no specific labels specified, noop.
+    if (dataframe.isEmpty()) {
+      if (!labels || srcLabels.length === 0) return this;
+      throw new Error("Empty dataframe, unable to pick columns");
+    }
+
+    if (this.isEmpty()) {
+      // 1. subset dataframe from source keys
+      // 2. alias names
+      dataframe = dataframe.subset(null, srcLabels);
+      for (let i = 0; i < srcLabels.length; i += 1) {
+        dataframe = dataframe.renameCol(srcLabels[i], dstLabels[i]);
+      }
+      return dataframe;
+    }
+
+    // otherwise, bulid a new dataframe combining columns from both
+
+    const srcOffsets = srcLabels.map(l => dataframe.colIndex.getOffset(l));
+
+    // check for label collisions
+    if (dstLabels.some(this.hasCol, this)) {
+      throw new Error("duplicate key collision");
+    }
+
+    // const dims = [this.dims[0], this.dims[1] + dataframe.dims[1]];
+    const dims = [this.dims[0], this.dims[1] + srcOffsets.length];
    const { rowIndex } = this;
-    const columns = [...this.__columns, ...dataframe.__columns];
-    const colIndex = this.colIndex.withLabels(dataframe.colIndex.keys());
+    const columns = [
+      ...this.__columns,
+      ...srcOffsets.map(i => dataframe.__columns[i])
+    ];
+    const colIndex = this.colIndex.withLabels(dstLabels);
    const columnsAccessor = [
      ...this.__columnsAccessor,
-      ...dataframe.__columnsAccessor
+      ...srcOffsets.map(i => dataframe.__columnsAccessor[i])
    ];
+
    return new this.constructor(
      dims,
      columns,
@@ -373,6 +441,11 @@ class Dataframe {
    );
  }

+  withColsFromAll(dataframes = []) {
+    dataframes = Array.isArray(dataframes) ? dataframes : [dataframes];
+    return dataframes.reduce((acc, df) => acc.withColsFrom(df), this);
+  }
+
  dropCol(label) {
    /*
    Create a new dataframe, omitting one columns.
@@ -566,7 +639,7 @@ class Dataframe {

    withRowIndex allows assignment of new row index during subset operation.
    If withRowIndex === null, it will reset the index to identity (offset)
-    indexing.  if withRowIndex is a label index object, it will be used
+    indexing. If withRowIndex is a label index object, it will be used
    for the new dataframe.
    */
    return this.__subset(rowOffsets, colOffsets, withRowIndex);
@@ -0,0 +1,18 @@
+const maybeTruncateString = (str, maxLength) => {
+
+  let truncatedString = null;
+  if (
+    str.length > maxLength
+  ) {
+    truncatedString = `${str.slice(
+      0,
+      maxLength / 2
+    )}…${str.slice(
+      -maxLength / 2
+    )}`;
+  } 
+
+  return truncatedString || str;
+}
+
+export default maybeTruncateString;
@@ -1,5 +1,5 @@
 /*
-Helper functions for user-editable nnotations state management.
+Helper functions for user-editable annotations state management.
 See also reducers/annotations.js
 */
 import { unassignedCategoryLabel } from "../../globals";
@@ -11,6 +11,13 @@ import {
  diffexpDimensionName
 } from "../nameCreators";

+export function maxCategoryItems(config) {
+  return (
+    config.parameters?.["max-category-items"] ??
+    globals.configDefaults.parameters["max-category-items"]
+  );
+}
+
 /*
 Selection state for categoricals are tracked in an Object that
 has two main components for each category:
@@ -38,7 +45,7 @@ Remember that option values can be ANY js type, except undefined/null.
  }
 */
 function topNCategories(colSchema, summary, N) {
-  /* return top N by occurance in the data, preserving original category order */
+  /* return top N by occurrences in the data, preserving original category order */
  const { categories } = colSchema;
  const counts = categories.map(cat => summary.categoryCounts.get(cat) ?? 0);

@@ -51,26 +58,34 @@ function topNCategories(colSchema, summary, N) {
  );
  const topNindices = new Set(sortIndex.slice(0, N));

-  const topNCategories = [];
+  const _topNCategories = [];
  const topNCounts = [];
  for (let i = 0; i < categories.length; i += 1) {
    if (topNindices.has(i)) {
-      topNCategories.push(categories[i]);
+      _topNCategories.push(categories[i]);
      topNCounts.push(counts[i]);
    }
  }
-  return [topNCategories, topNCounts];
+  return [_topNCategories, topNCounts];
 }

-export function selectableCategoryNames(world, maxCategoryItems) {
-  const { schema } = world;
+export function selectableCategoryNames(schema, maxCatItems, names) {
+  /*
+  return all obs annotation names that are categorical AND have a
+  "reasonably" small number of categories AND are not the index column.
+
+  If the initial name list not provided, use everything in the schema.
+  */
+  if (!schema) return [];
  const { index, columns } = schema.annotations.obs;
+
  return columns
+    .filter(colSchema => !names || names.indexOf(colSchema.name) !== -1)
    .filter(colSchema => {
-      const { name, categories } = colSchema;
-      return (
-        categories && categories.length < maxCategoryItems && name !== index
-      );
+      const { type, name } = colSchema;
+      const isSelectableType =
+        type === "string" || type === "boolean" || type === "categorical";
+      return isSelectableType && name !== index;
    })
    .map(v => v.name);
 }
@@ -157,7 +172,7 @@ export function pruneVarDataCache(varData, needed) {
  */

  /*
-  VarDataCacheLowWatermark - this cofig value sets the minimum cache size,
+  VarDataCacheLowWatermark - this config value sets the minimum cache size,
  in columns, below which we don't throw away data.

  The value should be high enough so we are caching the maximum which will
@@ -185,3 +200,10 @@ export function pruneVarDataCache(varData, needed) {
  }
  return varData;
 }
+
+export function subsetAndResetGeneLists(state) {
+  const { userDefinedGenes, diffexpGenes } = state;
+  const newUserDefinedGenes = [].concat(userDefinedGenes, diffexpGenes).slice(0, globals.maxGenes);
+  const newDiffExpGenes = [];
+  return [newUserDefinedGenes, newDiffExpGenes];
+}
@@ -1,5 +1,8 @@
 /*
 Helpers for schema management
+
+TODO: all this would be much more natural if done with a framework
+like immutable.js
 */
 import _ from "lodash";

@@ -31,17 +34,8 @@ export function indexEntireSchema(schema) {
  return schema;
 }

-export function sortAllCategorical(schema) {
-  /* UI relies on OBS annotation categories being in presentation sort order */
-  schema.annotations.obs.columns.forEach(c => {
-    if (c.categories) {
-      c.categories = catLabelSort(c.writable, c.categories);
-    }
-  });
-}
-
-function _copy(schema) {
-  /* redux copy conventions - WARNING, only for modifyign obs annotations */
+function _copyObsAnno(schema) {
+  /* redux copy conventions - WARNING, only for modifying obs annotations */
  return {
    ...schema,
    annotations: {
@@ -51,7 +45,17 @@ function _copy(schema) {
  };
 }

-function _reindex(schema) {
+function _copyObsLayout(schema) {
+  return {
+    ...schema,
+    layout: {
+      ...schema.layout,
+      obs: _.cloneDeep(schema.layout.obs)
+    }
+  };
+}
+
+function _reindexObsAnno(schema) {
  /* reindex obs annotations ONLY */
  schema.annotations.obsByName = fromEntries(
    schema.annotations.obs.columns.map(v => [v.name, v])
@@ -59,18 +63,25 @@ function _reindex(schema) {
  return schema;
 }

+function _reindexObsLayout(schema) {
+  schema.layout.obsByName = fromEntries(
+    schema.layout.obs.map(v => [v.name, v])
+  );
+  return schema;
+}
+
 export function removeObsAnnoColumn(schema, name) {
-  const newSchema = _copy(schema);
+  const newSchema = _copyObsAnno(schema);
  newSchema.annotations.obs.columns = schema.annotations.obs.columns.filter(
    v => v.name !== name
  );
-  return _reindex(newSchema);
+  return _reindexObsAnno(newSchema);
 }

 export function addObsAnnoColumn(schema, name, defn) {
-  const newSchema = _copy(schema);
+  const newSchema = _copyObsAnno(schema);
  newSchema.annotations.obs.columns.push(defn);
-  return _reindex(newSchema);
+  return _reindexObsAnno(newSchema);
 }

 export function removeObsAnnoCategory(schema, name, category) {
@@ -82,7 +93,7 @@ export function removeObsAnnoCategory(schema, name, category) {
  const idx = categories.indexOf(category);
  if (idx === -1) throw new Error("category does not exist");

-  const newSchema = _reindex(_copy(schema));
+  const newSchema = _reindexObsAnno(_copyObsAnno(schema));

  /* remove category.  Do not need to resort as this can't change presentation order */
  newSchema.annotations.obsByName[name].categories.splice(idx, 1);
@@ -98,7 +109,7 @@ export function addObsAnnoCategory(schema, name, category) {
  const idx = categories.indexOf(category);
  if (idx !== -1) throw new Error("category already exists");

-  const newSchema = _reindex(_copy(schema));
+  const newSchema = _reindexObsAnno(_copyObsAnno(schema));

  /* add category, retaining presentation sort order */
  const catAnno = newSchema.annotations.obsByName[name];
@@ -108,3 +119,17 @@ export function addObsAnnoCategory(schema, name, category) {
  ]);
  return newSchema;
 }
+
+export function addObsLayout(schema, layout) {
+  /* add or replace a layout */
+  const newSchema = _copyObsLayout(schema);
+  newSchema.layout.obs.push(layout);
+  return _reindexObsLayout(newSchema);
+}
+
+export function removeObsLayout(schema, name) {
+  /* remove a layout */
+  const newSchema = _copyObsLayout(schema);
+  newSchema.layout.obs = schema.layout.obs.filter(v => v.name !== name);
+  return _reindexObsLayout(newSchema);
+}
@@ -1,11 +1,9 @@
-import _ from "lodash";
-
 import { unassignedCategoryLabel } from "../../globals";
 import { decodeMatrixFBS } from "./matrix";
 import * as Dataframe from "../dataframe";
 import { isFpTypedArray } from "../typeHelpers";
-import { indexEntireSchema, sortAllCategorical } from "./schemaHelpers";
-import { isCategoricalAnnotation } from "./annotationsHelpers";
+import { indexEntireSchema } from "./schemaHelpers";
+import catLabelSort from "../catLabelSort";

 /*
 Private helper function - create and return a template Universe
@@ -74,9 +72,9 @@ function promoteTypedArray(o) {
  return new TyepdArrayCtor(o);
 }

-function AnnotationsFBSToDataframe(arrayBuffer) {
+export function matrixFBSToDataframe(arrayBuffers) {
  /*
-  Convert a Matrix FBS to a Dataframe.
+  Convert array of Matrix FBS to a Dataframe.

  The application has strong assumptions that all scalar data will be
  stored as a float32 or float64 (regardless of underlying data types).
@@ -86,77 +84,41 @@ function AnnotationsFBSToDataframe(arrayBuffer) {
  All float data from the server is left as is.  All non-float is promoted
  to an appropriate float.
  */
-  const fbs = decodeMatrixFBS(arrayBuffer, true); // leave in place
-  const columns = fbs.columns.map(c => {
-    if (isFpTypedArray(c) || Array.isArray(c)) return c;
-    return promoteTypedArray(c);
-  });
-  const df = new Dataframe.Dataframe(
-    [fbs.nRows, fbs.nCols],
-    columns,
-    null,
-    new Dataframe.KeyIndex(fbs.colIdx)
-  );
-  return df;
-}
-
-function LayoutFBSToDataframe(arrayBuffer) {
-  const fbs = decodeMatrixFBS(arrayBuffer, true);
-  if (fbs.columns.length < 2 || !fbs.columns.every(isFpTypedArray)) {
-    // We have strong assumptions about the shape & type of layout data.
-    throw new Error("Unexpected layout data type returned from server");
+  if (!Array.isArray(arrayBuffers)) {
+    arrayBuffers = [arrayBuffers];
+  }
+  if (arrayBuffers.length === 0) {
+    return Dataframe.Dataframe.empty();
  }

+  const fbs = arrayBuffers.map(ab => decodeMatrixFBS(ab, true)); // leave in place
+  /* check that all FBS have same row dimensionality */
+  const { nRows } = fbs[0];
+  fbs.forEach(b => {
+    if (b.nRows !== nRows)
+      throw new Error("FBS with inconsistent dimensionality");
+  });
+  const columns = fbs
+    .map(fb =>
+      fb.columns.map(c => {
+        if (isFpTypedArray(c) || Array.isArray(c)) return c;
+        return promoteTypedArray(c);
+      })
+    )
+    .flat();
+  const colIdx = fbs.map(b => b.colIdx).flat();
+  const nCols = columns.length;
+
  const df = new Dataframe.Dataframe(
-    [fbs.nRows, fbs.nCols],
-    fbs.columns,
+    [nRows, nCols],
+    columns,
    null,
-    new Dataframe.KeyIndex(fbs.colIdx)
+    new Dataframe.KeyIndex(colIdx)
  );
  return df;
 }

-function reconcileSchemaCategoriesWithSummary(universe) {
-  /*
-  where we treat types as (essentially) categorical metadata, update
-  the schema with data-derived categories (in addition to those in
-  the server declared schema).
-
-  For example, boolean defined fields in the schema do not contain
-  explicit declaration of categories (nor do string fields).  In these
-  cases, add a 'categories' field to the schema so it is accessible.
-
-  In addition, we have a client-side convention (UI) that all writable
-  annotations must have an 'unassigned' category, even if it is not currently
-  in use.
-  */
-
-  universe.schema.annotations.obs.columns.forEach(s => {
-    if (
-      s.type === "string" ||
-      s.type === "boolean" ||
-      s.type === "categorical"
-    ) {
-      const categories = _.union(
-        s.categories ?? [],
-        universe.obsAnnotations.col(s.name).summarize().categories ?? []
-      );
-      s.categories = categories;
-    }
-
-    if (s.writable && s.categories.indexOf(unassignedCategoryLabel) === -1) {
-      s.categories = s.categories.concat(unassignedCategoryLabel);
-    }
-  });
-}
-
-export function createUniverseFromResponse(
-  configResponse,
-  schemaResponse,
-  annotationsObsResponse,
-  annotationsVarResponse,
-  layoutFBSResponse
-) {
+export function createUniverseFromResponse(configResponse, schemaResponse) {
  /*
  build & return universe from a REST 0.2 /config, /schema and /annotations/obs response
  */
@@ -167,41 +129,80 @@ export function createUniverseFromResponse(
  universe.schema = schema;
  universe.nObs = schema.dataframe.nObs;
  universe.nVar = schema.dataframe.nVar;
+
  /* add defaults, as we can't assume back-end will fully populate schema */
  if (!schema.layout.var) schema.layout.var = [];
  if (!schema.layout.obs) schema.layout.obs = [];
-
-  /* annotations */
-  universe.obsAnnotations = AnnotationsFBSToDataframe(annotationsObsResponse);
-  universe.varAnnotations = AnnotationsFBSToDataframe(annotationsVarResponse);
-  /* layout */
-  universe.obsLayout = LayoutFBSToDataframe(layoutFBSResponse);
-
-  /* sanity checks */
-  if (
-    universe.nObs !== universe.obsLayout.length ||
-    universe.nObs !== universe.obsAnnotations.length ||
-    universe.nVar !== universe.varAnnotations.length
-  ) {
-    throw new Error("Universe dimensionality mismatch - failed to load");
-  }
-
-  reconcileSchemaCategoriesWithSummary(universe);
-  sortAllCategorical(universe.schema);
  indexEntireSchema(universe.schema);
+  normalizeEntireSchema(universe.schema);

-  /* sanity checks */
-  if (
-    schema.annotations.obs.columns.some(
-      s => s.writable && !isCategoricalAnnotation(schema, s.name)
-    )
-  ) {
+  return universe;
+}
+
+function normalizeSchemaCategory(colSchema, col = undefined) {
+  const { type, writable } = colSchema;
+  if (type === "string" || type === "boolean" || type === "categorical") {
+    let categories = [
+      ...new Set([
+        ...(colSchema.categories ?? []),
+        ...(col?.summarize?.().categories ?? [])
+      ])
+    ];
+    if (writable && categories.indexOf(unassignedCategoryLabel) === -1) {
+      categories = categories.concat(unassignedCategoryLabel);
+    }
+    colSchema.categories = categories;
+  } else if (writable) {
    throw new Error(
      "Writable continuous obs annotations are not supported - failed to load"
    );
  }

-  return universe;
+  if (colSchema.categories) {
+    colSchema.categories = catLabelSort(writable, colSchema.categories);
+  }
+}
+
+function normalizeEntireSchema(schema) {
+  // currently only needed for obsAnnotations
+  schema.annotations.obs.columns.forEach(colSchema =>
+    normalizeSchemaCategory(colSchema)
+  );
+}
+
+export function addObsAnnotations(universe, df) {
+  const obsAnnotations = universe.obsAnnotations.withColsFromAll(df);
+  if (universe.nObs !== obsAnnotations.length) {
+    throw new Error("Universe dimensionality mismatch - failed to load");
+  }
+
+  // for all of the new data, reconcile with schema and sort categories.
+  const dfs = Array.isArray(df) ? df : [df];
+  const keys = dfs.map(d => d.colIndex.keys()).flat();
+  const { schema } = universe;
+  keys.forEach(k => {
+    const colSchema = schema.annotations.obsByName[k];
+    const col = obsAnnotations.col(k);
+    normalizeSchemaCategory(colSchema, col);
+  });
+
+  return { obsAnnotations, schema };
+}
+
+export function addVarAnnotations(universe, df) {
+  const varAnnotations = universe.varAnnotations.withColsFromAll(df);
+  if (universe.nVar !== varAnnotations.length) {
+    throw new Error("Universe dimensionality mismatch - failed to load");
+  }
+  return { varAnnotations };
+}
+
+export function addObsLayout(universe, df) {
+  const obsLayout = universe.obsLayout.withColsFromAll(df);
+  if (universe.nObs !== obsLayout.length) {
+    throw new Error("Universe dimensionality mismatch - failed to load");
+  }
+  return { obsLayout };
 }

 export function convertDataFBStoObject(universe, arrayBuffer) {
@@ -5,8 +5,8 @@ import { isContinuousAnnotation } from "./annotationsHelpers";

 /*

-World is a subset of universe.   Most code should use world, and should
-(generally) not use Universe.   World contains any per-obs or per-var data
+World is a subset of universe.  Most code should use world, and should
+(generally) not use Universe.  World contains any per-obs or per-var data
 that must be consistent acorss the app when we view/manipulate subsets
 of Universe.

@@ -113,7 +113,6 @@ function clipDataframe(

 /*
 Create World with contents eq entire universe.   Commonly used to initialize World.
-If clipQuantiles
 */
 export function createWorldFromEntireUniverse(universe) {
  const world = templateWorld();
@@ -253,28 +252,53 @@ function deduceDimensionType(attributes, fieldName) {
  return dimensionType;
 }

+function addObsDimension(crossfilter, world, anno) {
+  /*
+  add single dimension to the crosfilter
+  */
+  const { obsAnnotations } = world;
+  if (obsAnnotations.hasCol(anno.name)) {
+    const dimType = deduceDimensionType(anno, anno.name);
+    const colData = obsAnnotations.col(anno.name).asArray();
+    const name = obsAnnoDimensionName(anno.name);
+    if (dimType === "enum") {
+      return crossfilter.addDimension(name, "enum", colData);
+    }
+    if (dimType) {
+      return crossfilter.addDimension(name, "scalar", colData, dimType);
+    }
+  }
+  return crossfilter;
+}
+
+export function addObsDimensions(crossfilter, world) {
+  /*
+  Add to crossfilter any dimension present in world.obsAnnotations
+  but not yet in the crossfilter
+  */
+  const schema = world.schema.annotations.obsByName;
+  const dimsWeNeed = world.obsAnnotations.colIndex.keys();
+  crossfilter = dimsWeNeed.reduce((xfltr, name) => {
+    const dimName = obsAnnoDimensionName(name);
+    if (xfltr.hasDimension(dimName)) return xfltr;
+    return addObsDimension(xfltr, world, schema[name]);
+  }, crossfilter);
+  return crossfilter;
+}
+
 export function createObsDimensions(crossfilter, world, XYdimNames) {
  /*
  create and return a crossfilter with a dimension for every obs annotation
  for which we have a supported type, *except* for the index column, indicated
  by schema.annotations.obs.index.
  */
-  const { schema, obsLayout, obsAnnotations } = world;
+  const { schema, obsLayout } = world;
  const indexName = schema.annotations.obs.index;
  const annoList = schema.annotations.obs.columns.filter(
    anno => anno.name !== indexName
  );
  crossfilter = annoList.reduce((xfltr, anno) => {
-    const dimType = deduceDimensionType(anno, anno.name);
-    const colData = obsAnnotations.col(anno.name).asArray();
-    const name = obsAnnoDimensionName(anno.name);
-    if (dimType === "enum") {
-      return xfltr.addDimension(name, "enum", colData);
-    }
-    if (dimType) {
-      return xfltr.addDimension(name, "scalar", colData, dimType);
-    }
-    return xfltr;
+    return addObsDimension(xfltr, world, anno);
  }, crossfilter);

  return crossfilter.addDimension(
@@ -98,6 +98,7 @@ class BitArray {
      if (lowestZeroBit) {
        this.bitmask[col] |= lowestZeroBit;
        dim = 32 * col + BitArray.ctz(lowestZeroBit);
+        break;
      }
    }
    return dim;
@@ -0,0 +1,33 @@
+PROJECT_ROOT := $(shell git rev-parse --show-toplevel)
+PATH := $(PATH):$(PROJECT_ROOT)/scripts
+
+SHELL := env PATH='$(PATH)' /bin/bash
+
+# get_or_else_dev_env_default
+# - If a variable is defined, return its value
+# - Else return the default value from environment.dev
+define get_or_else_dev_env_default
+$(if $($(1)),$($(1)),$(shell sed -n 's/$(1)=\(.*\)/\1/p' < $(PROJECT_ROOT)/environment.default))
+endef
+
+export CXG_SERVER_PORT := $(call get_or_else_dev_env_default,CXG_SERVER_PORT)
+export CXG_CLIENT_PORT := $(call get_or_else_dev_env_default,CXG_CLIENT_PORT)
+export JEST_ENV := $(call get_or_else_dev_env_default,JEST_ENV)
+
+export CXG_SERVER_PORT
+export CXG_CLIENT_PORT
+export JEST_ENV
+
+# copy the client assests to a location known to the server
+# $(1) is the source of the client assets
+# $(2) is the destination
+define copy_client_assets
+	mkdir -p $(2)/common/web/static/img
+	mkdir -p $(2)/common/web/static/js
+	mkdir -p $(2)/common/web/templates/
+	cp $(1)/index.html $(2)/common/web/templates/
+	cp -r $(1)/static $(2)/common/web/
+	cp $(1)/favicon.png $(2)/common/web/static/img
+	cp $(1)/service-worker.js $(2)/common/web/static/js/
+endef
+
@@ -9,6 +9,12 @@

 **All instructions are expected to be run from the top level cellxgene directory unless otherwise specified.**

+### Environment
+
+For all `make` commands, `common.mk` automatically checks whether required environment variables are set and, if they are not set, assigns them default values from `environment.default`.
+
+You can set these environment variables manually with the `export` shell command, as in `export JEST_ENV=debug`.
+
 ## Running test suite
 Client and server tests run on Travis CI for every push, PR, and commit to master on github. End to end tests run nightly on master only. 

@@ -18,21 +24,19 @@ Steps to run the all unit tests:
 1. `make dev-env`
 1. `make unit-test`

+To run unit tests for the `client` code only:
+1. Start in the project root directory
+1. `cd client`
+1. `make unit-test`
+
 ### End to end tests

-End to end tests use two env variables:
-* `JEST_ENV` - environment to run end to end tests. Default `dev`
-    * `prod` - run headless with no slowdown, chromium will not open.
-    * `dev` - opens chromimum, runs tests with minimal slowdown, close on exit.
-    * `debug` - opens chromium, runs tests with 100ms slowdown, dev tools open, chrome stays open on exit.
-* `JEST_CXG_PORT` - port that end to end tests are being run on. Default `3000` (client hosted port).
+To run E2E tests, run `cd client` and `make smoke-test`

-On CI the end to end tests are run with `JEST_ENV` set to `prod` using the `smoke-test` make target.
-
-To run end to end tests as they will be run on CI use the following command:
-```
-JEST_ENV=prod JEST_CXG_PORT=5000 make pydist install-dist dev-env smoke-test
-```
+The `JEST_ENV` environment variable enables the following E2E test options:
+* `dev` - opens chromimum, runs tests with minimal slowdown, close on exit.
+* `debug` - opens chromium, runs tests with 100ms slowdown, dev tools open, chrome stays open on exit.
+* `prod` - run headless with no slowdown, chromium will not open.

 Run end to end tests interactively during development
 1. cellxgene should be installed as [specified in client dev](#install-1)
@@ -40,6 +44,11 @@ Run end to end tests interactively during development
 1. Run `npm run e2e` or `make e2e` from the `client` directory
 1. To debug a failing test `export JEST_ENV='debug'` and re-run.

+To run end to end tests _exactly_ as they will be run on CI use the following command:
+```
+JEST_ENV=prod make pydist install-dist dev-env smoke-test
+```
+
 ## Server dev
 ### Install
 * Build the client and put static files in place: `make build-for-server-dev`
@@ -1,56 +1,8 @@
-## Node scripts in package.json
+# Developer convenience methods

-#### backend-dev
+## Makefile

-**About** This scripts enables FE developers to run the rest API necessary to back the development server for the front end. It is intended to ensure that the FE developer gets the current version of the backend with a single command and no knowledge of python necessary. It creates and activates a virtual environment and installs cellxgene from the current branch.
-
-**Requires** Python3.6 - `virtual-env`, `pip`
-
-**Usage** `npm run backend-dev <dataset>` then you can launch the node development server to serve the current state of the FE `npm run start`
-
-**Tips** Developers will probably want to run this in parallel with the node dev server. You can either do this by running each in a separate termanial window or by running `backend-dev` in the background (add an `&` at the end of the command to run in the background: `npm run backend-dev <dataset> &`).
-
-**Breakdown**
-
-| command                                  | purpose                                                   |
-| ---------------------------------------- | --------------------------------------------------------- |
-| python3.6 -m venv cellxgene              | creates cellxgene virtual environment                     |
-| source cellxgene/bin/activate            | activates virtual environment                             |
-| yes \| pip uninstall cellxgene \|\| true | uninstalls cellxgene (if installed)                       |
-| pip install -e ..                        | installs current local version of cellxgene               |
-| cellxgene launch                         | launches cellxgene (must supply dataset as last parameter |
-
-### Test scripts
-
-#### test
-
-**About** Run test locally. In order for this command to succeed you will need to give it a specific unit test to run. It won't pass if you run all tests as may be expected. This is because the unit tests and end to end (e2e) tests require different testing environments.
-
-**Usage** `npm run test <test file or pattern>`
-
-#### unit-test
-
-**About** Runs all unit tests. It excludes any tests in the e2e folder. This is used by travis to run unit tests.
-
-**Usage** `npm run unit-test`
-
-#### smoke-test
-
-**About** Starts backend development server and runs end to end tests. This is what travis runs. It depends on the `e2e` and the `start-server-for-test` node scripts. One starts the server, the other runs the tests. If developing a front-end feature and just checking if tests pass, this is probabaly the one you want to run.
-
-**Requirements** Must set env variable for port. In terminal `export JEST_ENV='dev'` `export JEST_CXG_PORT=5000`
-
-**Usage** `npm run smoke-test`
-
-#### e2e
-
-**About** Runs backend tests without starting the server. You will need to start the rest api separately with the pbmc3k.h5ad file.
-
-**Requirements** Must set env variable for port. In terminal `export JEST_ENV='dev'` `export JEST_CXG_PORT=5000`
-
-**Usage** `npm run e2e`
-
-## makefile
+Documentation for the `Makefile` targets in the project root directory.

 ### Build commands

@@ -91,7 +43,6 @@ Installs requirements files

 ```
 dev-env - installs requirements and requirments-dev (for building code)
-gui-env - installs requirements and requirments-dev and requirments-gui (for building native app)
 ```

 ### install commands
@@ -107,14 +58,67 @@ install-dist - installs from local dist folder
 uninstall - uninstalls cellxgene
 ```

-### gui
+## Client Makefile

-Commands for building the native app
+The following phony `make` targets in `client/Makefile` are convenience methods for getting you up and developing.

-```
-build-assets - builds the image and icon assets for the gui to pull from
-gui-spec-osx - creates the initial spec file for osx, do not run unless you are starting from scratch, one time only
-gui-spec-windows - creates the initial spec file for windows, do not run unless you are starting from scratch, one time only
-gui-build-osx - builds the app from the osx spec file
-gui-build-windows - builds the app from the windows spec file
-```
+### Runner scripts
+
+#### start-frontend
+
+**About** Serve the current client javascript independently from the `server` code.
+
+**Requires**
+* The server to be running. Best way to do this is with `make backend-dev`
+* `make ci` to install the necessary node modules
+
+**Usage** `make start-frontend`
+
+#### backend-dev
+
+**About** This script enables FE developers to run the REST API necessary to back the development server for the front end. It is intended to ensure that the FE developer gets the current version of the backend with a single command and no knowledge of python necessary. It creates and activates a virtual environment and installs cellxgene from the current branch.
+
+**Requires** Python3.6 - `virtual-env`, `pip`
+
+**Usage** `make backend-dev`. Optionally, you can then launch the node development server to serve the current state of
+the FE with `make start-frontend`. You can also select a specific dataset using `DATASET=<dataset path> make backend-dev`.
+You can also use `CXG_OPTIONS` to pass options to the `cellxgene launch` command, as in
+`CXG_OPTIONS='--experimental-annotations --experimental-annotations-file annotations.csv' make backend-dev`.
+
+**Tips** Developers will probably want to run this in parallel with the node dev server. You can either do this by running each in a separate termanial window or by running `backend-dev` in the background (add an `&` at the end of the command to run in the background: `DATASET=<dataset> make backend-dev &`).
+
+**Breakdown**
+
+| command                                  | purpose                                                   |
+| ---------------------------------------- | --------------------------------------------------------- |
+| python3.6 -m venv cellxgene              | creates cellxgene virtual environment                     |
+| source cellxgene/bin/activate            | activates virtual environment                             |
+| yes \| pip uninstall cellxgene \|\| true | uninstalls cellxgene (if installed)                       |
+| pip install -e .                        | installs current local version of cellxgene               |
+| cellxgene launch                         | launches cellxgene (must supply dataset as last parameter) |
+
+### Test scripts
+
+#### test
+
+**About** Run test locally. In order for this command to succeed you will need to give it a specific unit test to run. It won't pass if you run all tests as may be expected. This is because the unit tests and end to end (e2e) tests require different testing environments.
+
+**Usage** `make test`
+
+#### unit-test
+
+**About** Runs all unit tests. It excludes any tests in the e2e folder. This is used by travis to run unit tests.
+
+**Usage** `make unit-test`
+
+#### smoke-test
+
+**About** Starts backend development server and runs end to end tests. This is what travis runs. It depends on the `e2e` and the `backend-dev` targets. One starts the server, the other runs the tests. If developing a front-end feature and just checking if tests pass, this is probabaly the one you want to run.
+
+**Usage** `make smoke-test`
+
+#### e2e
+
+**About** Runs backend tests without starting the server. You will need to start the rest api separately with the pbmc3k.h5ad file. Note you can use the `JEST_ENV` environment variable to change how JEST runs in the browser.
+
+**Usage** `make e2e`
@@ -13,7 +13,7 @@ We are _piloting_ a new feature in cellxgene that enables users to create and ed

 To get started, run:
 ```
-cellxgene launch mydata.h5ad --experimental-annotations
+cellxgene launch mydata.h5ad
 ```

 To preserve data provenance, **`cellxgene` does not alter the input h5ad file**.  Rather, newly-created annotations are saved in a specified CSV file:  
@@ -21,8 +21,7 @@ To preserve data provenance, **`cellxgene` does not alter the input h5ad file**.
 annotations-directory/name-########.csv
 ```

- The default `annotations-directory` is  your current working directory (i.e., the directory you were in when you started cellxgene).
-
+- The default `annotations-directory` is your current working directory (i.e., the directory you were in when you started cellxgene).
 - You will be prompted to enter a name for your annotations the first time you create a new category.  
 - We also assign a unique identifier in the form of an 8-character suffix, `########`; this helps cellxgene identify your file to avoid overwriting your work.

@@ -36,10 +35,7 @@ There are two options for updating draft annotations.

 ### Autodetect annotations csv

-Cellxgene can automatically find and reload your draft annotations in editable mode like so:  
-```
-annotations-directory$ cellxgene launch mydata.h5ad --experimental-annotations
-```
+Cellxgene will automatically find and reload your draft annotations in editable mode.

 This assumes that:
 1 - The h5ad filename is the same  
@@ -52,7 +48,7 @@ This assumes that:

 If you'd like to specify the complete file path for your annotations, you can do so by running:  
 ```
-cellxgene launch mydata.h5ad --experimental-annotations --experimental-annotations-file path/to/myfile.csv
+cellxgene launch mydata.h5ad --annotations-file path/to/myfile.csv
 ```

 Any changes you make will be reflected in the original CSV. If the file does not exist, it will be created.  
@@ -83,7 +79,7 @@ Specifying a single file name for multiple contributors will result in data over

 To specify an output directory, run:  
 ```
-cellxgene launch mydata.h5ad --experimental-annotations --experimental-annotations-output-dir path/to/annotations-directory/
+cellxgene launch mydata.h5ad --annotations-dir path/to/annotations-directory/
 ```

 For each user, annotations will be saved as follows:  
@@ -93,8 +89,6 @@ For each user, annotations will be saved as follows:



-
-
 ## FAQ

 ### How do I know my annotations are saved?
@@ -113,4 +107,4 @@ This is most likely because the h5ad file you are working with is not the origin
 We place a small cookie (file) in your browser that identifies where your draft annotations are saved. This file never leaves your machine, and is never sent to the cellxgene team or anyone else.

 ### I have feedback and ideas for you!
-Wonderful! This is a very new and complex feature; we would _love_ to [hear your feedback](contact) :)
+Wonderful! This is a relatively new feature; we would _love_ to [hear your feedback](contact) :)
@@ -1,4 +1,5 @@
 # Code of conduct
+
 We warmly welcome contributions from the community!

 To ensure a welcoming experience for our entire community, this project adheres to the Contributor Covenant
@@ -13,12 +14,16 @@ If you have any questions about any of this stuff, just ask! :)
 We'd love to hear from you! Please submit any bug reports and feature requests through [Github issues](https://github.com/chanzuckerberg/cellxgene/issues).

 # Direct contributions
+
 ## Getting started

-If you are interested in working on `cellxgene` development, you'll need to use git to make a copy of the [project repository](https://www.youtube.com/watch?v=A-4WltCTVms&list=PLe6EXFvnTV7-_41SpakZoTIYCgX4aMTdU&index=2&t=0s) and share your changes.
-If you're new to git, we recommend [GitKraken](https://www.gitkraken.com/) for an intuitive interface.
+If you are interested in working on `cellxgene` development, you'll need to use git to make a copy of the [project repository](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/working-with-forks) and share your changes.

-Please submit any direct contributions by [forking the repository](https://www.youtube.com/watch?v=Lb4yvfrX_7I&list=PLe6EXFvnTV7-_41SpakZoTIYCgX4aMTdU&index=3&t=9s), creating a branch, and [submitting a Pull Request](https://www.youtube.com/watch?v=2VX1ISk9XH8&list=PLe6EXFvnTV7-_41SpakZoTIYCgX4aMTdU&index=9&t=0s).
+We have several "rules" (strong suggestions really) for contributions:
+
+1. If your contribution is complex, adds new features, new UI design or otherwise warrants discussion, we highly recommend that you submit a github issue, and engage other contributors in a discussion about the details of your proposed PR. This will save you time in the long run, as many details and decisions can be hashed out ahead-of-time.
+
+2. Please submit any direct contributions by [forking the repository](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/working-with-forks), creating a feature branch, and [submitting a Pull Request](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request).

 First, you'll need the following installed on your machine

@@ -33,13 +38,13 @@ git clone https://github.com/chanzuckerberg/cellxgene.git

 This is enough to get you started with editing documentation. If you'd like to contribute code:

-Build the client web assets by calling `make` from inside the `cellxgene` folder
+Build the client web assets from inside the `cellxgene` folder

 ```
-make
+make build-for-server-dev
 ```

-Install all requirements (we recommend doing this inside a [virtual environment](install))
+Install all requirements (we recommend doing this inside a [virtual environment](https://docs.python.org/3/tutorial/venv.html))

 ```
 pip install -e .
@@ -23,9 +23,9 @@ The following configuration options require special consideration in any multi-u

 `--disable-diffexp`: the differential expression computation can be resource intensive, in particular for large datasets. If many differential expression calculation requests are made in rapid sequence, it may cause the server CPU or memory resources to be exhausted, and impact the ability of other users to access data. This command line option will disable the differential expression feature, including the removal of the `Differential expression` button.

-`--experimental-annotations`: this feature, which is disabled by default, may not be appropriate for hosted environments. It will write to the local file system, and in extreme cases could be used to abuse (or exceed) file system capacity on the hosting server.
+`--disable-annotations`: annotations, which is enabled by default, may not be appropriate for hosted environments. It will write to the local file system, and in extreme cases could be used to abuse (or exceed) file system capacity on the hosting server. We recommend disabling this with this flag.

-`--experimental-annotations-file`: this specifies a single file for all end-user annotations, and is incompatible with hosted or multi-user use of cellxgene. Using it will cause loss of user annotation data (ie, the CSV file will be overwritten). If you wish to explore using the _experimental_ annotations feature in a multi-user environment, please refer to the [annotations documentation](annotations).
+`--annotations-file`: this specifies a single file for all end-user annotations, and is incompatible with hosted or multi-user use of cellxgene. Using it will cause loss of user annotation data (ie, the CSV file will be overwritten). If you wish to explore using the annotations feature in a multi-user environment, please refer to the [annotations documentation](annotations).

 ## Community software projects

@@ -42,7 +42,7 @@ If you know of other solutions, drop us a note and we'll add to this list.

 Clicking on the following button will forward you to Heroku to begin the deployment process:

-<a href="https://heroku.com/deploy?template=https://github.com/chanzuckerberg/cellxgene/tree/heroku">
+<a href="https://heroku.com/deploy?template=https://github.com/chanzuckerberg/cellxgene">
 <img src="https://www.herokucdn.com/deploy/button.svg" alt="Deploy">
 </a>

@@ -46,7 +46,7 @@ cellxgene launch s3://mybucket.s3-us-west-2.amazonaws.com/mydata.h5ad

 `--open` automatically opens the web browser after launching (caveat: only works on some operating systems).

-`--experimental-annotations`, `--experimental-annotations-file` & `--experimental-annotations-output-dir` all have to do with an experimental feature to allow users to create new categorical annotations in the application. We have a [whole separate page](annotations) about their usage! :)
+`--disable-annotations`, `--annotations-file` & `--annotations-dir` all have to do with creating new categorical annotations in the application. We have a [whole separate page](annotations) about their usage! :)

 `--diffexp-lfc-cutoff` as explained [in the methods](methods), genes are only returned in differential expression if the effect size is above the specified threshold for log fold change. Defaults to 0.01.

@@ -0,0 +1,4 @@
+# Default environment variables for cellxgene development
+CXG_SERVER_PORT=5005
+CXG_CLIENT_PORT=3000
+JEST_ENV=dev
@@ -4,7 +4,7 @@

 Clicking on the following button will forward you to Heroku to begin the deployment process:

-<a href="https://heroku.com/deploy?template=https://github.com/chanzuckerberg/cellxgene/tree/heroku">
+<a href="https://heroku.com/deploy?template=https://github.com/chanzuckerberg/cellxgene">
 <img src="https://www.herokucdn.com/deploy/button.svg" alt="Deploy">
 </a>

@@ -2,4 +2,4 @@ build:
  docker:
    web: experiments/heroku/Dockerfile
 run:
-  web: curl -o dataset.h5ad $DATASET && cellxgene launch dataset.h5ad --host 0.0.0.0 --port $PORT
+  web: cellxgene launch $DATASET --host 0.0.0.0 --port $PORT
@@ -17,9 +17,8 @@ exclude = '''
    | buck-out
    | build
    | dist
-    | server/app/util/fbs/NetEncoding
+    | server/data_common/fbs/NetEncoding
  )/
-  | server/gui/cellxgene_rc.py

 )
 '''
@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+import argparse
+import errno
+import socket
+import sys
+import time
+
+# await_port
+#
+# Functionality:
+# 1. Wait for a process to be listening on a port - await_port will poll the address with a bind request and close it
+#    until another process takes the address.
+# 2. Wait for a port to be free - await_port will poll the address with a bind request and close it until the port is
+#    free.
+
+parser = argparse.ArgumentParser(description='Wait for a port either to be bound to or to be free.')
+parser.add_argument('port', type=int, help='port to wait for')
+parser.add_argument('--sleep-increment', type=int, default=2.0, help='sleep duration in between checks of the port')
+parser.add_argument('--timeout', type=int, default=60.0, help='How long should we wait before timing out?')
+parser.add_argument('--await-free', default=False, action='store_true',
+                    help='Wait for the port to be active or free?')
+
+args = parser.parse_args()
+
+start = time.time()
+
+port_in_use = None
+
+desired_state = 'free' if args.await_free else 'in use'
+
+
+def condition_reached(port_in_use):
+    return not port_in_use if args.await_free else port_in_use
+
+
+while True:
+    # halt if we've reached the timeout
+    elapsed = time.time() - start
+    if elapsed > args.timeout:
+        print(f"Timed out while waiting for port {args.port}")
+        sys.exit(1)
+
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        try:
+            s.bind(('localhost', args.port))
+            port_in_use = False
+        except socket.error as e:
+            if e.errno != errno.EADDRINUSE:
+                raise e
+            port_in_use = True
+
+    if condition_reached(port_in_use):
+        break
+
+    print(f"Waiting for port {args.port} to be {desired_state}... sleeping for {args.sleep_increment} seconds.")
+    time.sleep(args.sleep_increment)
+
+print(f"Port {args.port} is {desired_state}!")
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+set -e
+
+# start_server_and_test
+# Replicates the basic functionality of the start-server-and-test npm package
+# https://www.npmjs.com/package/start-server-and-test
+
+if [[ $# -ne 3 ]]; then
+  echo "Usage: start_server_and_test \'[start server script]\' [port] \'[test script]\'"
+  exit 1
+fi
+
+START_SERVER_SCRIPT="$1"
+PORT="$2"
+TEST_SCRIPT="$3"
+
+await_port --await-free "$PORT"
+
+eval "$START_SERVER_SCRIPT &"
+SERVER_PID=$!
+
+function finish {
+  kill $SERVER_PID || true
+}
+
+trap finish EXIT
+
+await_port "$PORT"
+
+eval "$TEST_SCRIPT"
@@ -1,8 +1,17 @@
+include ../common.mk
+
 .PHONY: clean
 clean:
-	rm -f app/web/templates/index.html
-	rm -rf app/web/static
+	rm -f common/web/templates/index.html
+	rm -rf common/web/static

 .PHONY: unit-test
 unit-test:
-	pytest -s test
+	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
+		--source=app,cli,common,compute,converters,data_anndata,data_common,data_cxg \
+		--omit=.coverage,data_common/fbs/NetEncoding,venv \
+		-m unittest discover \
+		--start-directory test/ \
+		--top-level-directory ../ \
+		--verbose
+
@@ -1 +1 @@
-__version__ = "0.14.1"
+__version__ = "0.15.0"
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Severiano Badajoz	9c2f57df39	bump version to 0.15.0 (#1222 )	2020-03-13 13:11:12 -07:00
Bruce Martin	9bc83fdd02	update contributor process recommendations (#1213 ) * update contrib process recommendations * fix typos * clarification * fix build instructions * updates suggested by Matt * fix dead link	2020-03-13 11:41:30 -07:00
bmccandless	bbfd2e5ac5	Add a simple annotations cache to the server (#1220 ) Reading the user annotations had a performance problem. The same file was read over and over. Now, the last set of user annotations are cached.	2020-03-12 20:06:44 -07:00
Bruce Martin	f28bd52d79	fix performance regression caused by excessive rendering (#1219 )	2020-03-12 15:05:08 -07:00
Bruce Martin	63f1f38f7c	use AnchorButton so the tooltip dismisses (#1217 )	2020-03-12 08:36:36 -07:00
Severiano Badajoz	3ba4cf4483	fix bug created from master rebase (#1216 )	2020-03-11 15:20:20 -07:00
Matt Weiden	a862f2b2a4	Further refine compatibility tests (#1215 ) * Make sure apt is up to date before pulling hdf5 * Only install py dev reqs in cxg release vs anndata master test * Don't need bu flag when using sed on ubuntu * Don't re-install package reqs in python x anndata ver tests * Minor documentation fix	2020-03-11 14:16:05 -07:00
Severiano Badajoz	3bf72829c2	pass empty coordinates when a continuous annotation is colored (#1212 )	2020-03-11 13:01:14 -07:00
Severiano Badajoz	7cd9a0032a	optimize centroid util (#1147 ) * move unvarying evaluations outside of loop * refactoring * remove perf checks * commenting * minor fix + renaming * small fix * benchmarking * merge master * Revert pref checks * renaming and comment * remove redundant sets in Map	2020-03-11 12:35:42 -07:00
Sidney BellandMatt Weiden	9089fc98f2	annotations cli updates (#1190 ) * Switch logic from `--annotations` to `--disable-annotations * Rename `--annotations-file` --> `--annotations-input-file` and remove `experimental` * update docs * update makefile * update tests * Disable annotations on standard client smoke test * Update docs/posts/annotations.md Co-Authored-By: Matt Weiden <538456+mweiden@users.noreply.github.com> * Update docs/posts/annotations.md * Renaming * Docs * Update tests * Pesky typo -_- Co-authored-by: Matt Weiden <538456+mweiden@users.noreply.github.com>	2020-03-11 11:57:52 -07:00
Bruce Martin	8beeb57c96	Use DataLocator consistently on backend (#1208 ) * allow DataLocator to accept another locator as init param * migrate to DataLocator * migrate to DataLocator * lint * migrate to DataLocator * add check for erroroneous use of remote path and annotations * lint * revert default data location - now back go CWD * remove unused import	2020-03-10 19:12:49 -07:00
Colin MegillandMatt Weiden	234f25b782	Conditionally truncate category string (#1206 ) * maybe truncate string * add string formatting to test * correct import * destructuring * add maxlength * test * Respond to feedback from @bkmartinjr Co-authored-by: Matt Weiden <538456+mweiden@users.noreply.github.com>	2020-03-10 18:14:26 -07:00
Bruce Martin	28ddb60b2f	update JS deps (#1203 )	2020-03-10 10:59:12 -07:00
Colin Megill	1a93068810	Prevent category collapsing on annotation edit, # of cells on add label button (#1204 ) * collapse functionality * expand collapse	2020-03-10 13:21:06 -04:00
Matt Weiden	d8231a9a3b	Fixes to compatibility tests (#1205 ) * Unpin anndata ver in tests against anndata master * Add branch config to work on push * Add links to tests in GitHub Actions status badges * Remove temporary branch name	2020-03-10 08:19:36 -07:00
Bruce Martin	ffdcc24a40	remove pinned tables dependency (#1202 )	2020-03-10 07:47:54 -07:00
Severiano Badajoz	023a73cab8	smoke-test centroids + graph overlay layer (#1189 ) * add test labels * prettier + add clcik return to clickOn() * prettier + begin test * finish label counting test * add util to get coordinates of element * add test id to labels * add test to check overlay transform * remove logs * rename to match master	2020-03-09 17:01:22 -07:00
Bruce Martin	144b19c449	experimental re-embedding (#1186 ) * first cut at re-embedding route and back-end support * update and expand config route tests * add scanpy_umap * add reembedding to config route parameters * front-end support for reembedding fetch and UI * remove unused imports * add loading state * save reembedding in reducer state * improve withColsFrom * transmit reembed schema to client; pick unique embedding names * display embeddings * format * lint * spaces, tab size 2 * lint * test hack for smoke-test race * back out hack sleep * add check for backed mode * add unit test for reembedding * lint * hide re-embedding CLI param from help	2020-03-09 16:53:30 -07:00
bmccandlessandbkmartinjr	b3e9719602	hosted cellxgene (#38 ) (#1200 ) * early, non-working eb config * hosted cellxgene In this PR, contains scripts and instructions for deploying cellxgene for AWS elastic beanstalk. It supports the multi-dataset option. The Makefile in the server/eb directory creates an artifact.zip file, which can be deploy at AWS EB. The server/eb directory contains: app.py - flask app to run the server Makefile - which creates an artifact.zip file which can be deployed. README.md - instructions for setting up and deploying the eb app. * hosted cellxgene (#38) In this PR, contains scripts and instructions for deploying cellxgene for AWS elastic beanstalk. It supports the multi-dataset option. The Makefile in the server/eb directory creates an artifact.zip file, which can be deploy at AWS EB. The server/eb directory contains: app.py - flask app to run the server Makefile - which creates an artifact.zip file which can be deployed. README.md - instructions for setting up and deploying the eb app. * Update how artifact.zip is created prune the server/test and server/eb directories * Remove debugging print statements * fixes from review comments * fix lint Co-authored-by: bkmartinjr <bruce@chanzuckerberg.com>	2020-03-09 12:08:07 -07:00
Matt Weiden	bdd69160e2	Do not over-prune var data cache (#1198 ) * Remove unused variables and imports * Simplify conditional * Fix typo * Do not overprune var data cache There is a bug in how the universe and world gene sets are constructed and passed to `ControlsHelpers.pruneVarDataCache` that causes the var data cache to be over-pruned. This commit fixes the issue. Consider the following example from the node console: ``` ❯ node Welcome to Node.js v13.5.0. Type ".help" for more information. > new Set([1], [2], [3]) Set(1) { 1 } ``` What we really want is the set `Set(3) { 1, 2, 3 }`, which can be constructed as: ``` > new Set([].concat([1], [2], [3])) Set(3) { 1, 2, 3 } ```	2020-03-05 18:04:14 -08:00
Matt Weiden	bddf66d62a	Use blueprint "disable" property to indicate disabled buttons (#1197 ) * Use blueprint "disable" property to indicate that colorby disabled * Disable subset and reset buttons when action is not possible	2020-03-05 09:46:41 -08:00
Matt Weiden	7b77bf4bdd	Make smoke tests faster, more stable (#1195 ) * Refactor smoke tests & utils for conciseness/style * Modularize test utilities	2020-03-04 16:11:32 -08:00
Severiano Badajoz	cc890fe391	fix url launching on Heroku deployments (#1196 ) * fix improper branch link * set app to use dataset link * temp repo change * revert to master * add options var * tweak option order * remove options config arg	2020-03-04 13:05:11 -08:00
Severiano Badajoz	62e944796b	enable camera interaction on centroids (#1160 ) * pass handleCanvasEvent to graphOverlayLayer * dont pass handlecanvasevent if not interactive	2020-03-04 11:31:30 -08:00
Matt Weiden	1547a864c2	Disable ColorBy button for truncated categories (#1191 ) * Disable ColorBy button for truncated categories Fixes https://github.com/chanzuckerberg/cellxgene/issues/1156 For categories that have more than 100 labels we truncate the labels in the UI, but still allowed users to ColorBy these categories. Coloring by these categories can cause browsers to get bogged down. This commit disables ColorBy for truncated categories. * Minor documentation spelling and typo fixes * Respond to feedback from @liaprins-czi * Respond to feedback from @colinmegill and @bkmartinjr	2020-03-04 09:19:12 -08:00
Matt Weiden	05323ae643	Undo selection appends genes from differential expression to user gene list (#1183 ) * Undo selection appends diffExp genes to user gene list Fixes https://github.com/chanzuckerberg/cellxgene/issues/1171 Need: When a user performs a differential expression from within a sub-selection (world) of the data and then resets the selection to all cells (universe), the differential expression results are no longer valid. Approach: * When the selection is reset, move the top (maxUserDefinedGenes - len(userDefinedGenes) from the differential expression results to the list of user defined genes * Raise maxUserDefinedGenes to 25 to give users more room and accommodate the extra genes transferred in from differential expression Other commits: * Choose different button icons * Add diff exp genes to user defined genes on subset too * Respond to feedback from @liaprins-czi and @bkmartinjr	2020-03-02 12:07:12 -08:00
bmccandless	87efcc0e99	Fix a few bugs from server refactor (#1188 ) - Fixes #1185 port command line argument was not handle correctly - Fixes #1184 was not catching the "DatasetAccessError" exception.	2020-03-02 11:55:57 -08:00
Bruce Martin	bf7d7342d5	Improve label picking (#1179 ) * add simple error message helper * port all label name pickers to use the new LabelInput component * use pure components where possible * cleanup * more cleanup * lint * change new label prompt	2020-02-28 15:46:40 -07:00
Matt Weiden	1e4381ab7f	Make the Clip button primary (blue) (#1181 ) Fixes https://github.com/chanzuckerberg/cellxgene/issues/1101 I can't reproduce the active/inactive issue described in #1101 The only change remaining in #1101 is to make the clip button blue. This commit makes that change.	2020-02-26 16:54:30 -08:00
Matt Weiden	8b0adcdfcc	Add codecov configuration (#1180 ) * Add codecov configuration * Modify workflow	2020-02-26 14:57:46 -08:00
Severiano Badajoz	0043308b52	add centroid util unit tests (#1172 ) * scaffold the test * create tests	2020-02-26 12:16:13 -08:00
Matt Weiden	8a62c9f027	Add undo/redo smoke tests for annotations (#1175 ) * Add undo/redo tests for annotations Fixes https://github.com/chanzuckerberg/cellxgene/issues/969 ... also refactor the tests for DRY. * Add done() * Make e2e annotations tests safer to concurrency * Add data-testclass for save state. * Simplify tests and make them dependent on save state	2020-02-26 12:10:03 -08:00
Matt Weiden	ded35c7252	Spelling fix in Compatibility Test workflow (#1178 )	2020-02-26 11:12:16 -08:00
bmccandless	46876e2fb1	Fix various bugs with CXG format at annotations (#1173 ) - Enable testing for this path - Fixes #1166 - Fixes #1167 - Fixes #1168	2020-02-24 18:19:39 -08:00
Bruce Martin	7bc58bba2b	undo/redo cleanup (#1165 ) * fix refactoring error which disabled annotation file clearing * fix undo behavior on add category label * fix various undo/redo bugs * remove logging * further refinement of annotation undo/redo and actions * address PR comment	2020-02-24 18:51:26 -07:00
Matt Weiden	5c70cc5bcd	Add codecov for code coverage of python and javascript (#1170 ) * Add codecov to Push Test workflow * Empty commit * Clear reports and tag each with flags * Tag code reports by test * Fix codecov tags * One more fix	2020-02-24 17:13:16 -08:00
Matt Weiden	281154f591	Add exclusions to flake8 config (#1169 ) This excludes directories which should not be linted. This 1. makes the lint test more representative of what is run in CI 2. is faster	2020-02-24 08:33:25 -08:00
Matt Weiden	c7f2032dd7	Add user-generated annotations tests to the server (#1164 ) * Add user-generated annotations tests to the server Partially completes https://github.com/chanzuckerberg/cellxgene/issues/969 * Auto-format python code * @skip_if: passing lambdas > than property strings * Respond to feedback from @bkmartinjr	2020-02-23 15:32:13 -08:00
Bruce Martin	fb1f0c6469	fix latent crossfilter bug (#1162 )	2020-02-21 14:14:41 -07:00
Bruce Martin	d2bae0c4f6	CXG creator and dataset parameters (#1163 ) * remove redundant call to make_response (lint) * initial commit of H5AD to CXG converter * lint * PR review suggested comments * cxg adaptor now understands group metadata * lint	2020-02-21 12:49:45 -07:00
bmccandless	907cc634f5	server refactor (#1140 ) This PR contains a refactoring to make adding new features easier. The new features include supporting the tiledb format, and the multi dataset application. The refactoring includes Simplifying the directory structure and files. a class structure to handle annotations (currently one type: AnnotationsLocalFile). a class to handle application configuration a class structure to handle matrix data (currently AnndataAdaptor and CxgAdaptor). CxgAdaptor uses tiledb. Algorithms that were previously dependent on the scanpy anndata object are now generalized to work with an abstract interface. The multi dataset option is not fully supported yet, and so the option to use it is hidden. Use "cli launch --dataroot ..." To access this feature. All combinations of app single dataset/ app multi dataset and AnndataAdaptor/CxgAdaptor work with all the features, such as annotations, ontologies, diffexp.	2020-02-19 10:22:35 -08:00
Bruce Martin	349c413d8b	anndata X indexing & version compatibility improvements (#1157 ) * revert MatrixProxy; replace with correct use of adata slicing * work around 0.6 adata slicing bug * fix incorrect var slice * simplify slicing of X * add warning about performance impact of anndata<=0.7 * lint and remove unused code * improve comment * lint * correctly parse versions * temp files should preserve file suffix if possible - anndata 0.7 compat * update anndata dependency to 0.6.20 * resolve PR review comments	2020-02-19 09:57:51 -07:00
Matt Weiden	c630be33df	Simplify CI checks on git push, add nightly checks on cron (#1158 ) * Switch push tests to Github Actions * Add nightly tests * Add anndata check * Add test status badges * Add anndata 0.6.22.post1 workaround * Rename Nightly tests to Compatibility Tests * Fix to anndata fix * Turn on cron schedule * Fix badge mardown in README.md * Fix cron schedule	2020-02-18 11:57:19 -08:00
Matt Weiden	d30087ca63	Add frontend smoke tests for annotations (#1154 ) * Add smoke test for annotations features * Do not save during annotations tests * Fix botched rebase in dev guidelines * Revert "Do not save during annotations tests" This reverts commit `f0bd970bb2`. * Respond to feedback from @bkmartinjr	2020-02-14 17:23:45 -08:00
Matt Weiden	ac13b31e13	Collect all env vars in one, easy-to-find place (#1149 ) * Collect all env vars in one, easy-to-find place Past state: * Default environement variables were stored in both client/package.json and client/__tests__/e2e/config.js * Constants that should have been linked--like the cellxgene server port during testing--were repeated. With this commit: * All environment variables are parameterized * All environment variables are packaged in default env files * Move npm scripts to client Makefile * Respond to feedback from @seve and @bkmartinjr	2020-02-12 12:50:48 -08:00
Severiano Badajoz	5340a5f951	graph overlay improvements (#1141 ) * check to see if display state has changed * add display state * create onDisplayChange * check to see if displaying anything and add opacity drop * pass callback down to children * add middle truncation to labels * remove unused import * add a bit of documentation * make prop addition more clear * rename onDisplayChange -> overlayToggled for readability	2020-02-12 10:56:41 -08:00
Bruce Martin	e770db1e2c	load annotations incrementally (#1107 ) * load annotations individually * fix type check to be more general * update node CI version from 10 to 12 * node 11 * debug print node version * travis node version to latest * try nvm * remove extraneous node_js statement * remove node version debugging printf * incrementally load all annotations and layout * process annotations and layout as they are loaded * fix tests * sort categories incrementally * incrementally build category view summary; add category loading spinner * add spinner to continuous metadata * configure undoable reducer * incremental crossfilter creation * improve busy layout * more layout cleanup * correctly reconcile categories in schema * refine layout of lsb spinners * more spinner layout work * more spinner layout * always load layout before obs annotations	2020-02-10 11:22:27 -08:00
Bruce Martin	1c9b9f6a08	js package dep update (#1145 ) * js package dep update * roll back start-server-and-test	2020-02-05 13:11:05 -08:00
Bruce Martin	3afc03b95b	fix state prop bug with camera transform (#1138 )	2020-01-31 12:13:35 -08:00
Sidney Bell	a927d733c2	[WIP] Patch for anndata 0.7 (#1132 ) * Fix JSON encoding bug introduced by anndata 0.7 * Add anndata csr and csc classes to allowed matrix proxy inputs	2020-01-28 16:12:43 -08:00