bump version to 0.15.0 (#1222)

* update contrib process recommendations

* fix typos

* clarification

* fix build instructions

* updates suggested by Matt

* fix dead link

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.14.1
+current_version = 0.15.0
 
 [bumpversion:file:setup.py]
 search = version="{current_version}"

.codecov.yml

@@ -0,0 +1,8 @@
+coverage:
+  status:
+    project:
+      default:
+        informational: true
+    patch:
+      default:
+        informational: true

.github/workflows/compatibility_tests.yml

@@ -0,0 +1,118 @@
+name: Compatibility Tests
+
+on:
+  schedule:
+    - cron: '0 8 7 * 2'
+  push:
+    branches:
+      - master
+
+env:
+  JEST_ENV: prod
+
+jobs:
+  docker-build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Build docker image
+        run: docker build .
+
+  cellxgene-master-with-python-and-anndata-versions:
+    name: python versions x anndata versions
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8]
+        anndata-version: [0.6.22.post1, 0.7.1]
+        test-suite: [smoke-test, smoke-test-annotations]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        # 1. only install the dev requirements on top of what is in the cellxgene pip package
+        sudo apt-get update && sudo apt-get install -y libhdf5-serial-dev
+        sed -i 's/-r requirements.txt//' server/requirements-dev.txt
+        pip install -r server/requirements-dev.txt
+        # 2. install cellxgene
+        make pydist install-dist
+        # 3. install anndata
+        pip install anndata==${{ matrix.anndata-version }}
+        # workaround for anndata 0.6.22.post1 bug
+        [[ "0.6.22.post1" = "${{ matrix.anndata-version }}" ]] && pip install h5py==2.9.0 || true
+    - name: Tests
+      run: make unit-test ${{ matrix.test-suite }}
+
+  cellxgene-release-with-anndata-master:
+    name: cellxgene release with anndata master
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        test-suite: [smoke-test, smoke-test-annotations]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          path: cellxgene
+      - name: Checkout tools repo
+        uses: actions/checkout@v2
+        with:
+          repository: theislab/anndata
+          path: anndata
+      - name: Install dependencies
+        run: |
+          cd cellxgene
+          # 1. only install the dev requirements on top of what is in the cellxgene pip package
+          make dev-env-client
+          sed -i 's/-r requirements.txt//' server/requirements-dev.txt
+          pip install -r server/requirements-dev.txt
+          # 2. install cellxgene
+          pip install --upgrade cellxgene
+          # 3. install anndata
+          cd ../anndata && pip install -e .
+      - name: Tests
+        run: cd cellxgene && make unit-test ${{ matrix.test-suite }}
+
+  cellxgene-master-with-anndata-master:
+    name: cellxgene master with anndata master
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        test-suite: [smoke-test, smoke-test-annotations]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          path: cellxgene
+      - name: Checkout tools repo
+        uses: actions/checkout@v2
+        with:
+          repository: theislab/anndata
+          path: anndata
+      - name: Install dependencies
+        run: |
+          cd cellxgene
+          sed -i -E 's/^anndata[>=]=[0-9]+.[0-9]+.[0-9]+$/anndata/g' server/requirements.txt
+          make pydist install-dist dev-env
+          cd ../anndata
+          pip install -e .
+      - name: Tests
+        run: cd cellxgene && make unit-test ${{ matrix.test-suite }}

.github/workflows/push_tests.yml

@@ -0,0 +1,109 @@
+name: Push Tests
+
+on: [push]
+
+env:
+  JEST_ENV: prod
+  CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Lint with flake8
+      run: |
+        pip install flake8
+        make lint
+
+  unit-test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Python cache
+      uses: actions/cache@v1
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Node cache
+      uses: actions/cache@v1
+      with:
+        path: ~/.npm
+        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+        restore-keys: |
+          ${{ runner.os }}-node-
+    - name: Install dependencies
+      run: make pydist install-dist dev-env-server
+    - name: Unit tests
+      run: |
+        make unit-test
+        bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF backend,python,unitTest
+        cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
+
+  smoke-tests:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Python cache
+      uses: actions/cache@v1
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Node cache
+      uses: actions/cache@v1
+      with:
+        path: ~/.npm
+        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+        restore-keys: |
+          ${{ runner.os }}-node-
+    - name: Install dependencies
+      run: make pydist install-dist
+    - name: Smoke tests (without annotations feature)
+      run: |
+        cd client && make smoke-test
+        ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTest
+
+  smoke-tests-annotations:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Python cache
+      uses: actions/cache@v1
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Node cache
+      uses: actions/cache@v1
+      with:
+        path: ~/.npm
+        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+        restore-keys: |
+          ${{ runner.os }}-node-
+    - name: Install dependencies
+      run: make pydist install-dist
+    - name: Smoke tests (with annotations feature)
+      run: |
+        cd client && make smoke-test-annotations
+        ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTestAnnotations

.gitignore

@@ -3,6 +3,8 @@ node_modules/
 
 # coverage
 coverage
+coverage.xml
+.coverage
 .nyc_output
 
 .pytest_cache/
@@ -17,12 +19,12 @@ venv/
 cellxgene/
 
 # client build
-server/app/web/static/css/
+server/common/web/static/css/
-server/app/web/static/img/
+server/common/web/static/img/
-server/app/web/static/media/
+server/common/web/static/media/
-server/app/web/static/fonts/
+server/common/web/static/fonts/
-server/app/web/static/js/
+server/common/web/static/js/
-server/app/web/templates/index\.html
+server/common/web/templates/index\.html
 
 # Jupyter Notebook
 .ipynb_checkpoints

.travis.yml

@@ -1,28 +0,0 @@
-language: python
-dist: xenial
-sudo: required
-node_js:
-  - 10
-cache:
-  - pip
-  - npm
-install:
-  - set -eo pipefail
-  - pip install flake8
-  - make pydist install-dist dev-env
-
-jobs:
-  include:
-    - name: "Branch Tests 3.7"
-      python: "3.7"
-      script: make build-client lint unit-test
-    - name: "Branch Tests 3.6"
-      python: "3.6"
-      script: make build-client lint unit-test
-    - name: "Docker Build"
-      install: skip
-      python: "3.6"
-      script: docker build .
-    - name: "Smoke Tests"
-      python: "3.6"
-      script: make smoke-test

MANIFEST.in

@@ -1,5 +1,5 @@
-recursive-include server/app/web/templates *
+recursive-include server/common/web/templates *
-recursive-include server/app/web/static *
+recursive-include server/common/web/static *
 
 include server/requirements.txt
 include server/requirements-prepare.txt

Makefile

@@ -1,3 +1,5 @@
+include common.mk
+
 BUILDDIR := build
 CLIENTBUILD := $(BUILDDIR)/client
 SERVERBUILD := $(BUILDDIR)/server
@@ -5,7 +7,6 @@ CLEANFILES :=  $(BUILDDIR)/ client/build build dist cellxgene.egg-info
 
 PART ?= patch
 
-
 # CLEANING
 .PHONY: clean
 clean: clean-lite clean-server clean-client
@@ -33,25 +34,13 @@ build-client:
 build-cli: build-client
 	git ls-files server/ | cpio -pdm $(BUILDDIR)
 	cp -r client/build/  $(CLIENTBUILD)
-	mkdir -p $(SERVERBUILD)/app/web/static/img
+	$(call copy_client_assets,$(CLIENTBUILD),$(SERVERBUILD))
-	mkdir -p $(SERVERBUILD)/app/web/templates/
-	cp $(CLIENTBUILD)/index.html $(SERVERBUILD)/app/web/templates/
-	cp -r $(CLIENTBUILD)/static $(SERVERBUILD)/app/web/
-	cp $(CLIENTBUILD)/favicon.png $(SERVERBUILD)/app/web/static/img
-	cp $(CLIENTBUILD)/service-worker.js $(SERVERBUILD)/app/web/static/js/
 	cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR)
 
 # If you are actively developing in the server folder use this, dirties the source tree
 .PHONY: build-for-server-dev
 build-for-server-dev: clean-server build-client
-	mkdir -p server/app/web/static/img
+	$(call copy_client_assets,client/build,server)
-	mkdir -p server/app/web/static/js
-	mkdir -p server/app/web/templates/
-	cp client/build/index.html server/app/web/templates/
-	cp -r client/build/static server/app/web/
-	cp client/build/favicon.png server/app/web/static/img
-	cp client/build/service-worker.js server/app/web/static/js/
-
 
 # TESTING
 .PHONY: test
@@ -67,14 +56,20 @@ unit-test-%:
 smoke-test:
 	cd client && $(MAKE) smoke-test
 
+.PHONY: smoke-test-annotations
+smoke-test-annotations:
+	cd client && $(MAKE) smoke-test-annotations
+
 # FORMATTING CODE
 
 .PHOHY: fmt
 fmt: fmt-client fmt-py
 
+.PHONY: fmt-client
 fmt-client:
 	cd client && $(MAKE) fmt
 
+.PHONY: fmt
 fmt-py:
 	black .
 
@@ -120,13 +115,15 @@ release-directly-to-prod: dev-env pydist twine-prod
 	@echo "    make install-release"
 
 .PHONY: dev-env
-dev-env:
+dev-env: dev-env-client dev-env-server
-	cd client && $(MAKE) ci
-	pip install -r server/requirements-dev.txt
 
-.PHONY: gui-env
+.PHONY: dev-env-client
-gui-env: dev-env
+dev-env-client:
-	pip install -r server/requirements-gui.txt
+	cd client && $(MAKE) ci
+
+.PHONY: dev-env-server
+dev-env-server:
+	pip install -r server/requirements-dev.txt
 
 # give PART=[major, minor, part] as param to make bump
 .PHONY: bump
@@ -182,30 +179,3 @@ install-dist: uninstall
 uninstall:
 	pip uninstall -y cellxgene || :
 
-
-# GUI
-
-.PHONY: build-assets
-build-assets:
-	pyside2-rcc server/gui/cellxgene.qrc -o server/gui/cellxgene_rc.py
-
-.PHONY: gui-spec-osx
-gui-spec-osx: clean-lite gui-env
-	pip install -e .[gui]
-	pyi-makespec -D -w --additional-hooks-dir server/gui/ -n cellxgene  --add-binary='/System/Library/Frameworks/Tk.framework/Tk':'tk' --add-binary='/System/Library/Frameworks/Tcl.framework/Tcl':'tcl'  --add-data server/app/web/templates/:server/app/web/templates/ --add-data server/app/web/static/:server/app/web/static/ --icon server/gui/images/cxg_icons.icns server/gui/main.py
-	mv cellxgene.spec cellxgene-osx.spec
-
-.PHONY: gui-spec-windows
-gui-spec-windows: clean-lite dev-env
-	pip install -e .[gui]
-	pyi-makespec -D -w --additional-hooks-dir server/gui/ -n cellxgene --add-data server/app/web/templates;server/app/web/templates --add-data server/app/web/static;server/app/web/static --icon server/gui/images/icon.ico server/gui/main.py
-	mv cellxgene.spec cellxgene-windows.spec
-
-.PHONY: gui-build-osx
-gui-build-osx: clean-lite
-	pyinstaller --clean cellxgene-osx.spec
-
-.PHONY: gui-build-windows
-gui-build-windows: clean-lite
-	pyinstaller --clean cellxgene-windows.spec
-

README.md

@@ -3,6 +3,9 @@
 _an interactive explorer for single-cell transcriptomics data_
 
 [![DOI](https://zenodo.org/badge/105615409.svg)](https://zenodo.org/badge/latestdoi/105615409) [![PyPI](https://img.shields.io/pypi/v/cellxgene)](https://pypi.org/project/cellxgene/) [![PyPI - Downloads](https://img.shields.io/pypi/dm/cellxgene)](https://pypistats.org/packages/cellxgene) [![GitHub last commit](https://img.shields.io/github/last-commit/chanzuckerberg/cellxgene)](https://github.com/chanzuckerberg/cellxgene/pulse)
+[![Push Tests](https://github.com/chanzuckerberg/cellxgene/workflows/Push%20Tests/badge.svg)](https://github.com/chanzuckerberg/cellxgene/actions?query=workflow%3A%22Push+Tests%22)
+[![Compatibility Tests](https://github.com/chanzuckerberg/cellxgene/workflows/Compatibility%20Tests/badge.svg)](https://github.com/chanzuckerberg/cellxgene/actions?query=workflow%3A%22Compatibility+Tests%22)
+![Code Coverage](https://codecov.io/gh/chanzuckerberg/cellxgene/branch/master/graph/badge.svg)
 
 cellxgene (pronounced "cell-by-gene") is an interactive data explorer for single-cell transcriptomics datasets, such as those coming from the [Human Cell Atlas](https://humancellatlas.org). Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data.
 

app.json

@@ -1,7 +1,7 @@
 {
   "name": "cellxgene",
   "description": "An interactive explorer for single-cell transcriptomics data",
-  "repository": "https://github.com/chanzuckerberg/cellxgene/",
+  "repository": "https://github.com/chanzuckerberg/cellxgene",
   "logo": "https://cellxgene-example-data.czi.technology/favicon.png",
   "keywords": [
     "scientific",

cellxgene-osx.spec

@@ -1,41 +0,0 @@
-# -*- mode: python ; coding: utf-8 -*-
-
-block_cipher = None
-
-
-a = Analysis(['server/gui/main.py'],
-             pathex=['/Users/charlotteweaver/Documents/Git/cellxgene'],
-             binaries=[('/System/Library/Frameworks/Tk.framework/Tk', 'tk'), ('/System/Library/Frameworks/Tcl.framework/Tcl', 'tcl')],
-             datas=[('server/app/web/templates/', 'server/app/web/templates/'), ('server/app/web/static/', 'server/app/web/static/')],
-             hiddenimports=['sklearn', 'sklearn.utils._cython_blas', 'sklearn.neighbors.typedefs', 'sklearn.neighbors.quad_tree', 'sklearn.tree', 'sklearn.tree._utils'],
-             hookspath=['server/gui/'],
-             runtime_hooks=[],
-             excludes=[],
-             win_no_prefer_redirects=False,
-             win_private_assemblies=False,
-             cipher=block_cipher,
-             noarchive=False)
-pyz = PYZ(a.pure, a.zipped_data,
-             cipher=block_cipher)
-exe = EXE(pyz,
-          a.scripts,
-          [],
-          exclude_binaries=True,
-          name='cellxgene',
-          debug=False,
-          bootloader_ignore_signals=False,
-          strip=False,
-          upx=True,
-          console=False , icon='server/gui/images/cxg_icons.icns')
-coll = COLLECT(exe,
-               a.binaries,
-               a.zipfiles,
-               a.datas,
-               strip=False,
-               upx=True,
-               upx_exclude=[],
-               name='cellxgene')
-app = BUNDLE(coll,
-             name='cellxgene.app',
-             icon='server/gui/images/cxg_icons.icns',
-             bundle_identifier=None)

cellxgene-windows.spec

@@ -1,36 +0,0 @@
-# -*- mode: python -*-
-
-block_cipher = None
-
-
-a = Analysis(['server\\gui\\main.py'],
-             pathex=['C:\\Users\\Charlotte\\Documents\\git\\cellxgene'],
-             binaries=[],
-             datas=[('server/app/web/templates/', 'server/app/web/templates'), ('server/app/web/static/', 'server/app/web/static')],
-             hiddenimports=[],
-             hookspath=['server/gui/'],
-             runtime_hooks=[],
-             excludes=[],
-             win_no_prefer_redirects=False,
-             win_private_assemblies=False,
-             cipher=block_cipher,
-             noarchive=False)
-pyz = PYZ(a.pure, a.zipped_data,
-             cipher=block_cipher)
-exe = EXE(pyz,
-          a.scripts,
-          [],
-          exclude_binaries=True,
-          name='cellxgene',
-          debug=False,
-          bootloader_ignore_signals=False,
-          strip=False,
-          upx=True,
-          console=False , icon='server\\gui\\images\\icon.ico')
-coll = COLLECT(exe,
-               a.binaries,
-               a.zipfiles,
-               a.datas,
-               strip=False,
-               upx=True,
-               name='cellxgene')

client/Makefile

@@ -1,3 +1,10 @@
+include ../common.mk
+
+DATASET := $(if $(DATASET),$(DATASET),../example-dataset/pbmc3k.h5ad)
+ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../server/test/test_datasets/pbmc3k-annotations.csv)
+ANNOTATIONS_FILENAME := $(shell basename $(ANNOTATIONS))
+
+# Packaging
 .PHONY: clean
 clean:
 	rm -rf node_modules
@@ -14,7 +21,75 @@ install:
 build:
 	npm run build
 
+
+# Development convenience methods
+
+.PHONY: server-requirements
+server-requirements:
+	virtualenv -p python3 ../venv
+	source ../venv/bin/activate && \
+		pip install -r ../server/requirements-dev.txt && \
+		yes | pip uninstall cellxgene || true && \
+		pip install -e ..
+
+.PHONY: start-frontend
+start-frontend:
+	node server/development.js
+
+.PHONY: start-server
+start-server:
+	cellxgene launch -p $(CXG_SERVER_PORT) $(CXG_OPTIONS) $(DATASET)
+
+.PHONY: backend-dev
+backend-dev: server-requirements
+	source ../venv/bin/activate && $(MAKE) start-server
+
+.PHONY: backend-dev-anno-ontology
+backend-dev-anno-ontology: server-requirements
+	CXG_OPTIONS='--experimental-annotations-ontology' \
+		$(MAKE) backend-dev
+
+.PHONY: test
+test:
+	node node_modules/jest/bin/jest.js
+
+.PHONY: e2e
+e2e:
+	node node_modules/jest/bin/jest.js \
+		--verbose false \
+		--config __tests__/e2e/e2eJestConfig.json \
+		e2e/e2e.test.js
+
+.PHONY: e2e-annotations
+e2e-annotations:
+	node node_modules/jest/bin/jest.js \
+		--verbose false \
+		--config __tests__/e2e/e2eJestConfig.json \
+		e2e/e2eAnnotations.test.js
+
+.PHONY: smoke-test
+smoke-test:
+	start_server_and_test \
+		'CXG_OPTIONS="--disable-annotations" $(MAKE) start-server' \
+		$(CXG_SERVER_PORT) \
+		'$(MAKE) e2e'
+
+
+.PHONY: smoke-test-annotations
+smoke-test-annotations:
+	$(eval TMP_DIR := $(shell  mktemp -d /tmp/cellxgene_XXXXXX))
+	cp $(ANNOTATIONS) $(TMP_DIR)/ && \
+	start_server_and_test \
+		'CXG_OPTIONS="--annotations-file $(TMP_DIR)/$(ANNOTATIONS_FILENAME)" $(MAKE) start-server' \
+		$(CXG_SERVER_PORT) \
+		'$(MAKE) e2e-annotations'
+	rm -rf $(TMP_DIR)
+
+.PHONY: unit-test
+unit-test:
+	node node_modules/jest/bin/jest.js \
+	    --testPathIgnorePatterns e2e
+
 # pass remaining commands through to npm run
 %:
 	npm run $(*)
-

client/__tests__/e2e/cellxgeneActions.js

@@ -0,0 +1,196 @@
+import { strict as assert } from "assert";
+
+export const cellxgeneActions = (page, utils) => ({
+
+  async drag(testId, start, end, lasso = false) {
+    const layout = await utils.waitByID(testId);
+    const elBox = await layout.boxModel();
+    const x1 = elBox.content[0].x + start.x;
+    const x2 = elBox.content[0].x + end.x;
+    const y1 = elBox.content[0].y + start.y;
+    const y2 = elBox.content[0].y + end.y;
+    await page.mouse.move(x1, y1);
+    await page.mouse.down();
+    if (lasso) {
+      await page.mouse.move(x2, y1);
+      await page.mouse.move(x2, y2);
+      await page.mouse.move(x1, y2);
+      await page.mouse.move(x1, y1);
+    } else {
+      await page.mouse.move(x2, y2);
+    }
+    await page.mouse.up();
+  },
+
+  async clickOnCoordinate(testId, coord) {
+    const layout = await utils.waitByID(testId);
+    const elBox = await layout.boxModel();
+    const x = elBox.content[0].x + coord.x;
+    const y = elBox.content[0].y + coord.y;
+    await page.mouse.click(x, y);
+  },
+
+  async getAllHistograms(testclass, testIds) {
+    const histTestIds = testIds.map(tid => `histogram-${tid}`);
+    // these load asynchronously, so we need to wait for each histogram individually
+    await utils.waitForAllByIds(histTestIds);
+    const allHistograms = await utils.getAllByClass(testclass);
+    return allHistograms.map(hist => hist.replace(/^histogram-/, ""));
+  },
+
+  async getAllCategoriesAndCounts(category) {
+    await utils.waitByClass("categorical-row");
+    return page.$$eval(
+      `[data-testid="category-${category}"] [data-testclass='categorical-row']`,
+      rows => Object.fromEntries(rows.map(row => {
+        const cat = row.querySelector("[data-testclass='categorical-value']").innerText;
+        const count = row.querySelector("[data-testclass='categorical-value-count']").innerText;
+        return [cat, count];
+      }))
+    );
+  },
+
+  async cellSet(num) {
+    await utils.clickOn(`cellset-button-${num}`);
+    return utils.getOneElementInnerText(`[data-testid='cellset-count-${num}']`);
+  },
+
+  async resetCategory(category) {
+    const checkboxId = `${category}:category-select`;
+    await utils.waitByID(checkboxId);
+    const checkedPseudoclass = await page.$eval(
+      `[data-testid='${checkboxId}']`,
+      el => el.matches(":checked")
+    );
+    if (!checkedPseudoclass) await utils.clickOn(checkboxId);
+    try {
+      const categoryRow = await utils.waitByID(`${category}:category-expand`);
+      const isExpanded = await categoryRow.$("[data-testclass='category-expand-is-expanded']");
+      if (isExpanded) await utils.clickOn(`${category}:category-expand`);
+    } catch {}
+  },
+
+  async calcCoordinate(testId, xAsPercent, yAsPercent) {
+    const el = await utils.waitByID(testId);
+    const size = await el.boxModel();
+    return {
+      x: Math.floor(size.width * xAsPercent),
+      y: Math.floor(size.height * yAsPercent)
+    }
+  },
+
+  async calcDragCoordinates(testId, coordinateAsPercent) {
+    return {
+      start: await this.calcCoordinate(testId, coordinateAsPercent.x1, coordinateAsPercent.y1),
+      end: await this.calcCoordinate(testId, coordinateAsPercent.x2, coordinateAsPercent.y2)
+    };
+  },
+
+  async selectCategory(category, values, reset = true) {
+    if (reset) await this.resetCategory(category);
+    await utils.clickOn(`${category}:category-expand`);
+    await utils.clickOn(`${category}:category-select`);
+    for (const val of values) {
+      await utils.clickOn(`categorical-value-select-${category}-${val}`);
+    }
+  },
+
+  async expandCategory(category) {
+    const expand = await utils.waitByID(`${category}:category-expand`);
+    const notExpanded = await expand.$("[data-testclass='category-expand-is-not-expanded']");
+    if (notExpanded) await utils.clickOn(`${category}:category-expand`);
+  },
+
+  async clip(min = 0, max = 100) {
+    await utils.clickOn("visualization-settings");
+    await utils.clearInputAndTypeInto("clip-min-input", min);
+    await utils.clearInputAndTypeInto("clip-max-input", max);
+    await utils.clickOn("clip-commit");
+  },
+
+  async createCategory(categoryName) {
+    await utils.clickOn("open-annotation-dialog");
+    await utils.typeInto("new-category-name", categoryName);
+    await utils.clickOn("submit-category");
+  },
+
+  async renameCategory(oldCatgoryName, newCategoryName) {
+    await utils.clickOn(`${oldCatgoryName}:see-actions`);
+    await utils.clickOn(`${oldCatgoryName}:edit-category-mode`);
+    await utils.clearInputAndTypeInto(`${oldCatgoryName}:edit-category-name-text`, newCategoryName);
+    await utils.clickOn(`${oldCatgoryName}:submit-category-edit`);
+  },
+
+  async deleteCategory(categoryName) {
+    await utils.clickOn(`${categoryName}:see-actions`);
+    await utils.clickOn(`${categoryName}:delete-category`);
+  },
+
+  async createLabel(categoryName, labelName) {
+    await utils.clickOn(`${categoryName}:see-actions`);
+    await utils.clickOn(`${categoryName}:add-new-label-to-category`);
+    await utils.typeInto(`${categoryName}:new-label-name`, labelName);
+    await utils.clickOn(`${categoryName}:submit-label`);
+  },
+
+  async deleteLabel(categoryName, labelName) {
+    await this.expandCategory(categoryName);
+    await utils.clickOn(`${categoryName}:${labelName}:see-actions`);
+    await utils.clickOn( `${categoryName}:${labelName}:delete-label`);
+  },
+
+  async renameLabel(categoryName, oldLabelName, newLabelName) {
+    await this.expandCategory(categoryName);
+    await utils.clickOn(`${categoryName}:${oldLabelName}:see-actions`);
+    await utils.clickOn(`${categoryName}:${oldLabelName}:edit-label`);
+    await utils.clearInputAndTypeInto(
+      `${categoryName}:${oldLabelName}:edit-label-name`,
+      newLabelName
+    );
+    await utils.clickOn(`${categoryName}:${oldLabelName}:submit-label-edit`);
+  },
+
+  async addGeneToSearch(geneName) {
+    await utils.typeInto("gene-search", geneName);
+    await page.keyboard.press("Enter");
+    await page.waitForSelector(
+      `[data-testid='histogram-${geneName}']`
+    );
+  },
+
+  async subset(coordinatesAsPercent) {
+    // In order to deselect the selection after the subset, make sure we have some clear part
+    // of the scatterplot we can click on
+    assert(coordinatesAsPercent.x2 < 0.99 || coordinatesAsPercent.y2 < 0.99);
+    const lassoSelection = await this.calcDragCoordinates( "layout-graph", coordinatesAsPercent);
+    await this.drag("layout-graph", lassoSelection.start, lassoSelection.end, true );
+    await utils.clickOn("subset-button");
+    const clearCoordinate = await this.calcCoordinate(
+      "layout-graph",
+      0.5,
+      0.99
+    );
+    await this.clickOnCoordinate("layout-graph", clearCoordinate);
+  },
+
+  async setSellSet(cellSet, cellSetNum) {
+    for (const selection of cellSet.filter(sel => sel.kind === "categorical")) {
+      await this.selectCategory(selection.metadata, selection.values, true);
+    }
+    await this.cellSet(cellSetNum);
+  },
+
+  async runDiffExp(cellSet1, cellSet2) {
+    await this.setSellSet(cellSet1, 1);
+    await this.setSellSet(cellSet2, 2);
+    await utils.clickOn("diffexp-button");
+  },
+
+  async bulkAddGenes(geneNames) {
+    await utils.clickOn("section-bulk-add");
+    await utils.typeInto("input-bulk-add", geneNames.join(","));
+    await page.keyboard.press("Enter");
+  }
+});
+
+

client/__tests__/e2e/config.js

@@ -1,6 +1,9 @@
-export const jest_env = process.env.JEST_ENV || "dev";
+export const jest_env = process.env.JEST_ENV;
-export const appPort = process.env.JEST_CXG_PORT || 3000;
+export const appPort = process.env.CXG_SERVER_PORT;
 export const appUrlBase = `http://localhost:${appPort}`;
 export const DEV = jest_env === "dev";
 export const DEBUG = jest_env === "debug";
 export const DATASET = "pbmc3k";
+
+if (DEBUG) jest.setTimeout(100000);
+if (DEV) jest.setTimeout(10000);

client/__tests__/e2e/data.js

@@ -115,6 +115,25 @@ export const datasets = {
         }
       }
     },
+    categoryLabel: {
+      lasso: {
+        "coordinates-as-percent": { x1: 0.05, y1: 0.3, x2: 0.5, y2: 0.5 },
+      },
+      newCount: {
+        bySubsetConfig: {
+          false: "600",
+          true: "591"
+        }
+      }
+    },
+    annotationsFromFile: {
+      count: {
+        bySubsetConfig: {
+          false: "1161",
+          true: "856"
+        }
+      }
+    },
     clip: {
       min: "30",
       max: "70",

client/__tests__/e2e/e2e.test.js

@@ -3,45 +3,16 @@ Smoke test suite that will be run in Travis CI
 
 Tests included in this file are expected to be relatively stable and test core features
  */
-import puppeteer from "puppeteer";
+import { appUrlBase, DATASET } from "./config";
-import { appUrlBase, DEBUG, DEV, DATASET } from "./config";
+import { setupTestBrowser } from "./testBrowser";
-import { puppeteerUtils, cellxgeneActions } from "./puppeteerUtils";
 import { datasets } from "./data";
 
-let browser, page, utils, cxgActions, spy;
+let browser, page, utils, cxgActions;
-const browserViewport = { width: 1280, height: 960 };
+const data = datasets[DATASET];
-let data = datasets[DATASET];
-
-if (DEBUG) jest.setTimeout(100000);
-if (DEV) jest.setTimeout(10000);
 
 beforeAll(async () => {
-  const browserParams = DEV
+  const browserViewport = { width: 1280, height: 960 };
-    ? { headless: false, slowMo: 5 }
+  [browser, page, utils, cxgActions] = await setupTestBrowser(browserViewport);
-    : DEBUG
-    ? { headless: false, slowMo: 100, devtools: true }
-    : {};
-  browser = await puppeteer.launch(browserParams);
-  page = await browser.newPage();
-  await page.setViewport(browserViewport);
-  if (DEV || DEBUG) {
-    page.on("console", async msg => {
-      // If there is a console.error but an error is not thrown, this will ensure the test fails
-      if (msg.type() === "error") {
-        const errorMsgText = await Promise.all(
-          // TODO can we do this without internal properties?
-          msg.args().map(arg => arg._remoteObject.description)
-        );
-        throw new Error(`Console error: ${errorMsgText}`);
-      }
-      console.log(`PAGE LOG: ${msg.text()}`);
-    });
-  }
-  page.on("pageerror", err => {
-    throw new Error(`Console error: ${err}`);
-  });
-  utils = puppeteerUtils(page);
-  cxgActions = cellxgeneActions(page);
 });
 
 beforeEach(async () => {
@@ -49,15 +20,13 @@ beforeEach(async () => {
 });
 
 afterAll(() => {
-  if (!DEBUG) {
+  if (browser !== undefined) browser.close();
-    browser.close();
-  }
 });
 
 describe("did launch", () => {
   test("page launched", async () => {
-    let el = await utils.getOneElementInnerHTML("[data-testid='header']");
+    const element = await utils.getOneElementInnerHTML("[data-testid='header']");
-    expect(el).toBe(data.title);
+    expect(element).toBe(data.title);
   });
 });
 
@@ -65,11 +34,9 @@ describe("metadata loads", () => {
   test("categories and values from dataset appear", async () => {
     for (const label in data.categorical) {
       await utils.waitByID(`category-${label}`);
-      const categoryName = await utils.getOneElementInnerText(
+      const categoryName = await utils.getOneElementInnerText(`[data-testid="category-${label}"]`);
-        `[data-testid="category-${label}"]`
-      );
       expect(categoryName).toMatch(label);
-      await utils.clickOn(`category-expand-${label}`);
+      await utils.clickOn(`${label}:category-expand`);
       const categories = await cxgActions.getAllCategoriesAndCounts(label);
       expect(Object.keys(categories)).toMatchObject(
         Object.keys(data.categorical[label])
@@ -112,12 +79,10 @@ describe("cell selection", () => {
 
   test("selects cells via categorical", async () => {
     for (const cellset of data.cellsets.categorical) {
-      await utils.clickOn(`category-expand-${cellset.metadata}`);
+      await utils.clickOn(`${cellset.metadata}:category-expand`);
-      await utils.clickOn(`category-select-${cellset.metadata}`);
+      await utils.clickOn(`${cellset.metadata}:category-select`);
       for (const val of cellset.values) {
-        await utils.clickOn(
+        await utils.clickOn(`categorical-value-select-${cellset.metadata}-${val}`);
-          `categorical-value-select-${cellset.metadata}-${val}`
-        );
       }
       const cellCount = await cxgActions.cellSet(1);
       expect(cellCount).toBe(cellset.count);
@@ -139,46 +104,20 @@ describe("cell selection", () => {
 });
 
 describe("gene entry", () => {
-  test("search for single gene", async () => {
+  test("search for single gene", async () => cxgActions.addGeneToSearch(data.genes.search));
-    // blueprint's  typeahead is treating typing weird, clicking & waiting first solves this
-    await utils.typeInto("gene-search", data.genes.search);
-    await page.keyboard.press("Enter");
-    await page.waitForSelector(
-      `[data-testid='histogram-${data.genes.search}']`
-    );
-  });
 
   test("bulk add genes", async () => {
-    await cxgActions.reset();
     const testGenes = data.genes.bulkadd;
-    await utils.clickOn("section-bulk-add");
+    await cxgActions.bulkAddGenes(testGenes);
-    await utils.typeInto("input-bulk-add", testGenes.join(","));
+    const allHistograms = await cxgActions.getAllHistograms("histogram-user-gene", testGenes);
-    await page.keyboard.press("Enter");
-
-    const allHistograms = await cxgActions.getAllHistograms(
-      "histogram-user-gene",
-      testGenes
-    );
     expect(allHistograms).toEqual(expect.arrayContaining(testGenes));
     expect(allHistograms.length).toEqual(testGenes.length);
   });
 });
 
-describe("diffexp", () => {
+describe("differential expression", () => {
   test("selects cells, saves them and performs diffexp", async () => {
-    for (const select of data.diffexp.cellset1) {
+    await cxgActions.runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
-      if (select.kind === "categorical") {
-        await cxgActions.selectCategory(select.metadata, select.values, true);
-      }
-    }
-    await cxgActions.cellSet(1);
-    for (const select of data.diffexp.cellset2) {
-      if (select.kind === "categorical") {
-        await cxgActions.selectCategory(select.metadata, select.values, true);
-      }
-    }
-    await cxgActions.cellSet(2);
-    await utils.clickOn("diffexp-button");
     const allHistograms = await cxgActions.getAllHistograms(
       "histogram-diffexp",
       data.diffexp["gene-results"]
@@ -190,7 +129,7 @@ describe("diffexp", () => {
   });
 });
 
-describe("subset/reset", () => {
+describe("subset", () => {
   test("subset - cell count matches", async () => {
     for (const select of data.subset.cellset1) {
       if (select.kind === "categorical") {
@@ -209,39 +148,6 @@ describe("subset/reset", () => {
     }
   });
 
-  test("reset after subset", async () => {
-    for (const select of data.subset.cellset1) {
-      if (select.kind === "categorical") {
-        await cxgActions.selectCategory(select.metadata, select.values, true);
-      }
-    }
-    await utils.clickOn("subset-button");
-    for (const label in data.subset.categorical) {
-      const categories = await cxgActions.getAllCategoriesAndCounts(label);
-      expect(Object.keys(categories)).toMatchObject(
-        Object.keys(data.subset.categorical[label])
-      );
-      expect(Object.values(categories)).toMatchObject(
-        Object.values(data.subset.categorical[label])
-      );
-    }
-    await cxgActions.reset();
-    for (const label in data.categorical) {
-      await utils.waitByID(`category-${label}`);
-      const categoryName = await utils.getOneElementInnerText(
-        `[data-testid="category-${label}"]`
-      );
-      expect(categoryName).toMatch(label);
-      const categories = await cxgActions.getAllCategoriesAndCounts(label);
-      expect(Object.keys(categories)).toMatchObject(
-        Object.keys(data.categorical[label])
-      );
-      expect(Object.values(categories)).toMatchObject(
-        Object.values(data.categorical[label])
-      );
-    }
-  });
-
   test("lasso after subset", async () => {
     for (const select of data.subset.cellset1) {
       if (select.kind === "categorical") {
@@ -262,15 +168,49 @@ describe("subset/reset", () => {
     const cellCount = await cxgActions.cellSet(1);
     expect(cellCount).toBe(data.subset.lasso.count);
   });
+
+  test("undo selection appends the top diff exp genes to user defined genes", async () => {
+    const userDefinedGenes = data.genes.bulkadd;
+    const diffExpGenes = data.diffexp["gene-results"];
+    await cxgActions.bulkAddGenes(userDefinedGenes);
+    const userDefinedHistograms = await cxgActions.getAllHistograms("histogram-user-gene", userDefinedGenes);
+    expect(userDefinedHistograms).toEqual(expect.arrayContaining(userDefinedGenes));
+    await cxgActions.subset({x1: 0.15, y1: 0.10, x2: 0.98, y2: 0.98});
+    await cxgActions.runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
+    const diffExpHistograms = await cxgActions.getAllHistograms("histogram-diffexp", diffExpGenes);
+    expect(diffExpHistograms).toEqual(expect.arrayContaining(diffExpGenes));
+    await utils.clickOn("reset-subset-button");
+    const expected = [].concat(userDefinedGenes, diffExpGenes);
+    const userDefinedHistogramsAfterSubset = await cxgActions.getAllHistograms(
+      "histogram-user-gene",
+      expected
+    );
+    expect(userDefinedHistogramsAfterSubset).toEqual(expect.arrayContaining(expected));
+  });
+
+  test("subset selection appends the top diff exp genes to user defined genes", async () => {
+    const userDefinedGenes = data.genes.bulkadd;
+    const diffExpGenes = data.diffexp["gene-results"];
+    await cxgActions.bulkAddGenes(userDefinedGenes);
+    const userDefinedHistograms = await cxgActions.getAllHistograms("histogram-user-gene", userDefinedGenes);
+    expect(userDefinedHistograms).toEqual(expect.arrayContaining(userDefinedGenes));
+    await cxgActions.subset({x1: 0.15, y1: 0.10, x2: 0.98, y2: 0.98});
+    await cxgActions.runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
+    const diffExpHistograms = await cxgActions.getAllHistograms("histogram-diffexp", diffExpGenes);
+    expect(diffExpHistograms).toEqual(expect.arrayContaining(diffExpGenes));
+    await cxgActions.subset({x1: 0.16, y1: 0.11, x2: 0.97, y2: 0.97});
+    const expected = [].concat(userDefinedGenes, diffExpGenes);
+    const userDefinedHistogramsAfterSubset = await cxgActions.getAllHistograms(
+      "histogram-user-gene",
+      expected
+    );
+    expect(userDefinedHistogramsAfterSubset).toEqual(expect.arrayContaining(expected));
+  });
 });
 
 describe("scatter plot", () => {
   test("scatter plot appears", async () => {
-    await cxgActions.reset();
+    await cxgActions.bulkAddGenes(Object.values(data.scatter.genes));
-    const testGenes = data.scatter.genes;
-    await utils.clickOn("section-bulk-add");
-    await utils.typeInto("input-bulk-add", Object.values(testGenes).join(","));
-    await page.keyboard.press("Enter");
     await utils.clickOn(`plot-x-${data.scatter.genes.x}`);
     await utils.clickOn(`plot-y-${data.scatter.genes.y}`);
     await utils.waitByID("scatterplot");
@@ -338,6 +278,58 @@ describe("ui elements don't error", () => {
       panCoords.end,
       false
     );
-    await page.evaluate(`window.scrollBy(0, 1000);`);
+    await page.evaluate("window.scrollBy(0, 1000);");
+  });
+});
+
+describe("centroid labels", () => {
+  test("labels are created", async () => {
+    await utils.clickOn("centroid-label-toggle");
+    const labels = Object.keys(data.categorical);
+    /* eslint-disable no-await-in-loop */
+    // Toggle colorby for each category and check to see if labels are generated
+    for (let i = 0, { length } = labels; i < length; i += 1) {
+      const label = labels[i];
+      await utils.clickOn(`colorby-${label}`);
+      const generatedLabels = await utils.getAllByClass("centroid-label");
+      // Number of labels generated should be equal to size of the object
+      expect(generatedLabels).toHaveLength(
+        Object.keys(data.categorical[label]).length
+      );
+    }
+    /* eslint-enable no-await-in-loop */
+  });
+});
+
+describe("graph overlay", () => {
+  test("transform centroids correctly", async () => {
+    const category = Object.keys(data.categorical)[0];
+    await utils.clickOn("centroid-label-toggle");
+    await utils.clickOn(`colorby-${category}`);
+    await utils.clickOn("mode-pan-zoom");
+    const panCoords = await cxgActions.calcDragCoordinates(
+      "layout-graph",
+      data.pan["coordinates-as-percent"]
+    );
+
+    const categoryValue = Object.keys(data.categorical[category])[0];
+    const initialCoordinates = await utils.getElementCoordinates(
+      `${categoryValue}-centroid-label`
+    );
+    await cxgActions.drag(
+      "layout-graph",
+      panCoords.start,
+      panCoords.end,
+      false
+    );
+    const terminalCoordinates = await utils.getElementCoordinates(
+      `${categoryValue}-centroid-label`
+    );
+    expect(terminalCoordinates[0] - initialCoordinates[0]).toBeCloseTo(
+      panCoords.end.x - panCoords.start.x
+    );
+    expect(terminalCoordinates[1] - initialCoordinates[1]).toBeCloseTo(
+      panCoords.end.y - panCoords.start.y
+    );
   });
 });

client/__tests__/e2e/e2eAnnotations.test.js

@@ -0,0 +1,222 @@
+/*
+Tests included in this file are specific to annotation features
+*/
+import { appUrlBase, DATASET } from "./config";
+import { setupTestBrowser } from "./testBrowser";
+import { datasets } from "./data";
+
+let browser, page, utils, actions;
+const data = datasets[DATASET];
+
+beforeAll(async () => {
+  const browserViewport = {width: 1280, height: 960};
+  [browser, page, utils, actions] = await setupTestBrowser(browserViewport);
+});
+
+afterAll(() => {
+  if (browser !== undefined) browser.close()
+});
+
+describe.each([
+  {withSubset: true, tag: "subset"},
+  {withSubset: false, tag: "whole"}
+])("annotations", (config) => {
+
+  const perTestCategoryName = "per-test-category";
+  const perTestLabelName = "per-test-label";
+
+  beforeEach(async () => {
+    await page.goto(appUrlBase);
+    // wait for the page to load
+    await utils.waitByClass("autosave-complete");
+    // setup the test fixtures
+    await actions.createCategory(perTestCategoryName);
+    await actions.createLabel(perTestCategoryName, perTestLabelName);
+    if (config.withSubset) await actions.subset({x1: 0.10, y1: 0.10, x2: 0.80, y2: 0.80});
+    await utils.waitByClass("autosave-complete");
+  });
+
+  afterEach(async () => {
+    await deleteCategoryIfExists(perTestCategoryName);
+    await utils.waitByClass("autosave-complete");
+  });
+
+  test("create a category", async () => {
+    const categoryName = `category-created-${config.tag}`;
+    await assertCategoryDoesNotExist(categoryName);
+    await actions.createCategory(categoryName);
+    await assertCategoryExists(categoryName);
+  });
+
+  test("delete a category", async () => {
+    await actions.deleteCategory(perTestCategoryName);
+    await assertCategoryDoesNotExist(perTestCategoryName);
+  });
+
+  test("rename a category", async () => {
+    const newCategoryName = `cluster-for-real-${config.tag}`;
+    await actions.renameCategory(perTestCategoryName, newCategoryName);
+    await assertCategoryDoesNotExist(perTestCategoryName);
+    await assertCategoryExists(newCategoryName);
+  });
+
+  test("create a label", async () => {
+    const labelName = `new-label-${config.tag}`;
+    await assertLabelDoesNotExist(perTestCategoryName, labelName);
+    await actions.createLabel(perTestCategoryName, labelName);
+    await assertLabelExists(perTestCategoryName, labelName);
+  });
+
+  test("delete a label", async () => {
+    await actions.deleteLabel(perTestCategoryName, perTestLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName, perTestLabelName);
+  });
+
+  test("rename a label", async () => {
+    const newLabelName = "my-cool-new-label";
+    await assertLabelDoesNotExist(perTestCategoryName, newLabelName);
+    await actions.renameLabel(perTestCategoryName, perTestLabelName, newLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName, perTestLabelName);
+    await assertLabelExists(perTestCategoryName, newLabelName);
+  });
+
+  test("check cell count for a label loaded from file", async () => {
+    const categoryName = "cluster-test";
+    const labelName = "four";
+    await actions.expandCategory(categoryName);
+    const result = await utils.waitByID(`categorical-value-count-${categoryName}-${labelName}`);
+    expect(await result.evaluate(node => node.innerText)).toBe(
+      data.annotationsFromFile.count.bySubsetConfig[config.withSubset]
+    );
+  });
+
+  test("assign cells to a label", async () => {
+    await actions.expandCategory(perTestCategoryName);
+    const lassoSelection = await actions.calcDragCoordinates(
+      "layout-graph",
+      data.categoryLabel.lasso["coordinates-as-percent"]
+    );
+    await actions.drag(
+      "layout-graph",
+      lassoSelection.start,
+      lassoSelection.end,
+      true
+    );
+    await utils.waitByID("lasso-element", {visible: true});
+    await utils.clickOn(`${perTestCategoryName}:${perTestLabelName}:see-actions`);
+    await utils.clickOn(`${perTestCategoryName}:${perTestLabelName}:add-current-selection-to-this-label`);
+    const result = await utils.waitByID(`categorical-value-count-${perTestCategoryName}-${perTestLabelName}`);
+    expect(await result.evaluate(node => node.innerText)).toBe(
+      data.categoryLabel.newCount.bySubsetConfig[config.withSubset]
+    );
+  });
+
+  test("undo/redo category creation", async () => {
+    const categoryName = `category-created-undo-${config.tag}`;
+    await assertCategoryDoesNotExist(categoryName);
+    await actions.createCategory(categoryName);
+    await assertCategoryExists(categoryName);
+    await utils.clickOn("undo");
+    await assertCategoryDoesNotExist(categoryName);
+    await utils.clickOn("redo");
+    await assertCategoryExists(categoryName);
+  });
+
+  test("undo/redo category deletion", async () => {
+    const categoryName = `category-deleted-undo-${config.tag}`;
+    await actions.createCategory(categoryName);
+    await assertCategoryExists(categoryName);
+    await actions.deleteCategory(categoryName);
+    await assertCategoryDoesNotExist(categoryName);
+    await utils.clickOn("undo");
+    await assertCategoryExists(categoryName);
+    await utils.clickOn("redo");
+    await assertCategoryDoesNotExist(categoryName);
+  });
+
+  test("undo/redo category rename", async () => {
+    const newCategoryName = `category-renamed-undo-${config.tag}`;
+    await assertCategoryDoesNotExist(newCategoryName);
+    await actions.renameCategory(perTestCategoryName, newCategoryName);
+    await assertCategoryExists(newCategoryName);
+    await assertCategoryDoesNotExist(perTestCategoryName);
+    await utils.clickOn("undo");
+    await assertCategoryExists(perTestCategoryName);
+    await assertCategoryDoesNotExist(newCategoryName);
+    await utils.clickOn("redo");
+    await assertCategoryExists(newCategoryName);
+    await assertCategoryDoesNotExist(perTestCategoryName);
+  });
+
+  test("undo/redo label creation", async () => {
+    const labelName = `label-created-undo-${config.tag}`;
+    await assertLabelDoesNotExist(perTestCategoryName, labelName);
+    await actions.createLabel(perTestCategoryName, labelName);
+    await assertLabelExists(perTestCategoryName, labelName);
+    await utils.clickOn("undo");
+    await assertLabelDoesNotExist(perTestCategoryName);
+    await utils.clickOn("redo");
+    await assertLabelExists(perTestCategoryName, labelName);
+  });
+
+  test("undo/redo label deletion", async () => {
+    await actions.deleteLabel(perTestCategoryName, perTestLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName);
+    await utils.clickOn("undo");
+    await assertLabelExists(perTestCategoryName, perTestLabelName);
+    await utils.clickOn("redo");
+    await assertLabelDoesNotExist(perTestCategoryName);
+  });
+
+  test("undo/redo label rename", async () => {
+    const newLabelName = `label-renamed-undo-${config.tag}`;
+    await assertLabelDoesNotExist(perTestCategoryName, newLabelName);
+    await actions.renameLabel(perTestCategoryName, perTestLabelName, newLabelName);
+    await assertLabelExists(perTestCategoryName, newLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName, perTestLabelName);
+    await utils.clickOn("undo");
+    await assertLabelExists(perTestCategoryName, perTestLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName, newLabelName);
+    await utils.clickOn("redo");
+    await assertLabelExists(perTestCategoryName, newLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName, perTestLabelName);
+  });
+
+  async function assertCategoryExists(categoryName) {
+    const handle = await utils.waitByID(`${categoryName}:category-expand`);
+    const result = await handle.evaluate(node => node.innerText);
+    // slice beginning and end of category name result to account for truncation of long names
+    expect(result.slice(0, 10)).toBe(categoryName.slice(0, 10));
+    expect(result.slice(-10)).toBe(categoryName.slice(-10));
+  }
+
+  async function assertCategoryDoesNotExist(categoryName) {
+    const result = await page.$(`[data-testid='${categoryName}:category-expand']`);
+    expect(result).toBeNull();
+  }
+
+  async function assertLabelExists(categoryName, labelName) {
+    const category = await utils.waitByID(`${categoryName}:category-expand`);
+    expect(category).not.toBeNull();
+    await actions.expandCategory(categoryName);
+    const previous = await utils.waitByID(`categorical-value-${categoryName}-${labelName}`);
+    expect(await previous.evaluate(node => node.innerText)).toBe(labelName);
+  }
+
+  async function assertLabelDoesNotExist(categoryName, labelName) {
+    await actions.expandCategory(categoryName);
+    const result = await page.$(`[data-testid='categorical-value-${categoryName}-${labelName}']`);
+    expect(result).toBeNull();
+  }
+
+  async function deleteCategoryIfExists(categoryName) {
+    try {
+      const category = await page.waitForSelector(
+        `[data-testid='${categoryName}:category-expand']`,
+        {timeout: 200}
+      );
+      if (category !== null) return await actions.deleteCategory(categoryName);
+    } catch {}
+    return null
+  }
+});

client/__tests__/e2e/puppeteerUtils.js

@@ -1,211 +1,69 @@
-export const puppeteerUtils = puppeteerPage => ({
+export const puppeteerUtils = page => ({
-  async waitByID(testid, props = {}) {
+  async waitByID(testId, props = {}) {
-    return await puppeteerPage.waitForSelector(
+    return page.waitForSelector(`[data-testid='${testId}']`, props);
-      `[data-testid='${testid}']`,
-      props
-    );
   },
 
-  async waitByClass(testclass, props = {}) {
+  async waitByClass(testClass, props = {}) {
-    return await puppeteerPage.waitForSelector(
+    return page.waitForSelector(`[data-testclass='${testClass}']`, props);
-      `[data-testclass='${testclass}']`,
-      props
-    );
   },
 
-  async waitForAllByIds(testids, props = {}) {
+  async waitForAllByIds(testIds) {
     await Promise.all(
-      testids.map(testid =>
+      testIds.map(testId => page.waitForSelector(`[data-testid='${testId}']`))
-        puppeteerPage.waitForSelector(`[data-testid='${testid}']`)
-      )
     );
   },
 
-  async getAllByClass(testclass, props = {}) {
+  async getAllByClass(testClass) {
-    const elements = await puppeteerPage.$$eval(
+    return page.$$eval(`[data-testclass=${testClass}]`, eles =>
-      `[data-testclass=${testclass}]`,
+      eles.map(ele => ele.dataset.testid)
-      els => {
-        return els.map(el => {
-          return el.dataset.testid;
-        });
-      }
     );
-    return elements;
   },
 
-  async typeInto(testid, text) {
+  async typeInto(testId, text) {
+    // blueprint's  typeahead is treating typing weird, clicking & waiting first solves this
     // only works for text without special characters
-    await this.waitByID(testid);
+    await this.waitByID(testId);
-    const selector = `[data-testid='${testid}']`;
+    const selector = `[data-testid='${testId}']`;
     // type ahead can be annoying if you don't pause before you type
-    await puppeteerPage.click(selector);
+    await page.click(selector);
-    await puppeteerPage.waitFor(200);
+    await page.waitFor(200);
-    await puppeteerPage.type(selector, text);
+    await page.type(selector, text);
   },
 
-  async clearInputAndTypeInto(testid, text) {
+  async clearInputAndTypeInto(testId, text) {
-    await this.waitByID(testid);
+    await this.waitByID(testId);
-    const selector = `[data-testid='${testid}']`;
+    const selector = `[data-testid='${testId}']`;
     // only works for text without special characters
     // type ahead can be annoying if you don't pause before you type
-    await puppeteerPage.click(selector);
+    await page.click(selector);
-    await puppeteerPage.waitFor(200);
+    await page.waitFor(200);
     // select all
-
+    await page.click(selector, { clickCount: 3 });
-    await puppeteerPage.click(selector, { clickCount: 3 });
+    await page.keyboard.press("Backspace");
-    await puppeteerPage.keyboard.type("Backspace");
+    await page.type(selector, text);
-    await puppeteerPage.type(selector, text);
   },
 
-  async clickOn(testid) {
+  async clickOn(testid, options = {}) {
     await this.waitByID(testid);
-    await puppeteerPage.click(`[data-testid='${testid}']`);
+    const click = await page.click(`[data-testid='${testid}']`, options);
-    await puppeteerPage.waitFor(50);
+    await page.waitFor(50);
+    return click;
   },
 
   async getOneElementInnerHTML(selector) {
-    await puppeteerPage.waitForSelector(selector);
+    await page.waitForSelector(selector);
-    let text = await puppeteerPage.$eval(selector, el => el.innerHTML);
+    return page.$eval(selector, el => el.innerHTML);
-    return text;
   },
 
   async getOneElementInnerText(selector) {
-    await puppeteerPage.waitForSelector(selector);
+    await page.waitForSelector(selector);
-    let text = await puppeteerPage.$eval(selector, el => el.innerText);
+    return page.$eval(selector, el => el.innerText);
-    return text;
-  }
-});
-
-export const cellxgeneActions = puppeteerPage => ({
-  async drag(testid, start, end, lasso = false) {
-    const layout = await puppeteerUtils(puppeteerPage).waitByID(testid);
-    const elBox = await layout.boxModel();
-    const x1 = elBox.content[0].x + start.x;
-    const x2 = elBox.content[0].x + end.x;
-    const y1 = elBox.content[0].y + start.y;
-    const y2 = elBox.content[0].y + end.y;
-    await puppeteerPage.mouse.move(x1, y1);
-    await puppeteerPage.mouse.down();
-    if (lasso) {
-      await puppeteerPage.mouse.move(x2, y1);
-      await puppeteerPage.mouse.move(x2, y2);
-      await puppeteerPage.mouse.move(x1, y2);
-      await puppeteerPage.mouse.move(x1, y1);
-    } else {
-      await puppeteerPage.mouse.move(x2, y2);
-    }
-    await puppeteerPage.mouse.up();
-  },
-
-  async getAllHistograms(testclass, testids) {
-    const histTestIds = testids.map(tid => `histogram-${tid}`);
-    // these load asynchronously, so we need to wait for each histogram individually
-    await puppeteerUtils(puppeteerPage).waitForAllByIds(histTestIds);
-    const allHistograms = await puppeteerUtils(puppeteerPage).getAllByClass(
-      testclass
-    );
-    return allHistograms.map(hist =>
-      hist.substr("histogram_".length, hist.length)
-    );
-  },
-
-  async getAllCategoriesAndCounts(category) {
-    await puppeteerUtils(puppeteerPage).waitByClass("categorical-row");
-    const categories = await puppeteerPage.$$eval(
-      `[data-testid="category-${category}"] [data-testclass='categorical-row']`,
-      els => {
-        let result = {};
-        els.forEach(el => {
-          const cat = el.querySelector("[data-testclass='categorical-value']")
-            .innerText;
-          const count = el.querySelector(
-            "[data-testclass='categorical-value-count']"
-          ).innerText;
-          result[cat] = count;
-        });
-        return result;
-      }
-    );
-    return categories;
-  },
-
-  async cellSet(num) {
-    await puppeteerUtils(puppeteerPage).clickOn(`cellset-button-${num}`);
-    return await puppeteerUtils(puppeteerPage).getOneElementInnerText(
-      `[data-testid='cellset-count-${num}']`
-    );
-  },
-
-  async resetCategory(category) {
-    const checkboxId = `category-select-${category}`;
-    await puppeteerUtils(puppeteerPage).waitByID(checkboxId);
-    const checkedPseudoclass = await puppeteerPage.$eval(
-      `[data-testid='${checkboxId}']`,
-      el => {
-        return el.matches(":checked");
-      }
-    );
-    if (!checkedPseudoclass) {
-      await puppeteerUtils(puppeteerPage).clickOn(checkboxId);
-    }
-    try {
-      const categoryRow = await puppeteerUtils(puppeteerPage).waitByID(
-        `category-expand-${category}`
-      );
-      const isExpanded = await categoryRow.$(
-        "[data-testclass='category-expand-is-expanded']"
-      );
-      if (isExpanded) {
-        await puppeteerUtils(puppeteerPage).clickOn(
-          `category-expand-${category}`
-        );
-      }
-    } catch {}
-  },
-
-  async calcDragCoordinates(testid, coordinateAsPercent) {
-    const el = await puppeteerUtils(puppeteerPage).waitByID(testid);
-    const size = await el.boxModel();
-    const coords = {
-      start: {
-        x: Math.floor(size.width * coordinateAsPercent.x1),
-        y: Math.floor(size.height * coordinateAsPercent.y1)
-      },
-      end: {
-        x: Math.floor(size.width * coordinateAsPercent.x2),
-        y: Math.floor(size.height * coordinateAsPercent.y2)
-      }
-    };
-    return coords;
-  },
-
-  async selectCategory(category, values, reset = true) {
-    if (reset) await this.resetCategory(category);
-    await puppeteerUtils(puppeteerPage).clickOn(`category-expand-${category}`);
-    await puppeteerUtils(puppeteerPage).clickOn(`category-select-${category}`);
-    for (const val of values) {
-      await puppeteerUtils(puppeteerPage).clickOn(
-        `categorical-value-select-${category}-${val}`
-      );
-    }
-  },
-
-  async reset() {
-    await puppeteerUtils(puppeteerPage).clickOn("reset");
-    // loading state never actually happens, reset is too fast
-    await page.waitFor(200);
   },
 
-  async clip(min = 0, max = 100) {
+  async getElementCoordinates(testid) {
-    await puppeteerUtils(puppeteerPage).clickOn("visualization-settings");
+    return page.$eval(`[data-testid='${testid}']`, elem => {
-    await puppeteerUtils(puppeteerPage).clearInputAndTypeInto(
+      const { left, top } = elem.getBoundingClientRect();
-      "clip-min-input",
+      return [left, top];
-      min
+    });
-    );
-    await puppeteerUtils(puppeteerPage).clearInputAndTypeInto(
-      "clip-max-input",
-      max
-    );
-    await puppeteerUtils(puppeteerPage).clickOn("clip-commit");
   }
 });

client/__tests__/e2e/testBrowser.js

@@ -0,0 +1,35 @@
+import puppeteer from "puppeteer";
+import { DEBUG, DEV } from "./config";
+import { puppeteerUtils } from "./puppeteerUtils";
+import { cellxgeneActions } from "./cellxgeneActions";
+
+export async function setupTestBrowser(browserViewport) {
+  const browserParams = DEV
+    ? { headless: false, slowMo: 5 }
+    : DEBUG
+      ? { headless: false, slowMo: 100, devtools: true }
+      : {};
+  const browser = await puppeteer.launch(browserParams);
+  const page = await browser.pages().then(pages => pages[0]);
+  await page.setViewport(browserViewport);
+  if (DEV || DEBUG) {
+    page.on("console", async msg => {
+      // If there is a console.error but an error is not thrown, this will ensure the test fails
+      if (msg.type() === "error") {
+        const errorMsgText = await Promise.all(
+          // TODO can we do this without internal properties?
+          msg.args().map(arg => arg._remoteObject.description)
+        );
+        throw new Error(`Console error: ${errorMsgText}`);
+      }
+      console.log(`PAGE LOG: ${msg.text()}`);
+    });
+  }
+  page.on("pageerror", err => {
+    throw new Error(`Console error: ${err}`);
+  });
+  const utils = puppeteerUtils(page);
+  const cxgActions = cellxgeneActions(page, utils);
+  return [browser, page, utils, cxgActions];
+}
+

client/__tests__/util/centroid.test.js

@@ -0,0 +1,96 @@
+import _ from "lodash";
+
+import calcCentroid from "../../src/util/centroid";
+
+import quantile from "../../src/util/quantile";
+import * as Universe from "../../src/util/stateManager/universe";
+import * as World from "../../src/util/stateManager/world";
+import * as REST from "./stateManager/sampleResponses";
+import { ControlsHelpers as CH } from "../../src/util/stateManager";
+
+describe("centroid", () => {
+  let world;
+  let categoricalSelection;
+  beforeAll(() => {
+    // Create world + universe
+    let universe = Universe.createUniverseFromResponse(
+      _.cloneDeep(REST.config),
+      _.cloneDeep(REST.schema)
+    );
+
+    universe = {
+      ...universe,
+      ...Universe.addObsAnnotations(
+        universe,
+        Universe.matrixFBSToDataframe(REST.annotationsObs)
+      ),
+      ...Universe.addVarAnnotations(
+        universe,
+        Universe.matrixFBSToDataframe(REST.annotationsVar)
+      ),
+      ...Universe.addObsLayout(
+        universe,
+        Universe.matrixFBSToDataframe(REST.layoutObs)
+      )
+    };
+    world = World.createWorldFromEntireUniverse(universe);
+
+    // Create categorical selection from world
+    categoricalSelection = CH.createCategoricalSelection(
+      world,
+      CH.selectableCategoryNames(world.schema, CH.maxCategoryItems(REST.config))
+    );
+  });
+
+  test("field4 (categorical obsAnnotation)", () => {
+    const centroidResult = calcCentroid(
+      world.obsAnnotations,
+      world.obsLayout,
+      "field4",
+      ["umap_0", "umap_1"],
+      categoricalSelection,
+      world.schema.annotations.obsByName
+    );
+
+    // Check to see that a centroid has been calculated for every categorical value
+    const keysAsArray = Array.from(centroidResult.keys());
+    expect(keysAsArray).toEqual(
+      expect.arrayContaining([83, true, "foo", 2.222222])
+    );
+
+    // This expected result assumes that all cells belong in all categorical values inside of sample response
+    const expectedResult = [
+      quantile([0.5], world.obsLayout.col("umap_0").asArray())[0],
+      quantile([0.5], world.obsLayout.col("umap_1").asArray())[0]
+    ];
+
+    centroidResult.forEach(coordinate => {
+      expect(coordinate).toEqual(expectedResult);
+    });
+  });
+
+  test("field3 (boolean obsAnnotation", () => {
+    const centroidResult = calcCentroid(
+      world.obsAnnotations,
+      world.obsLayout,
+      "field3",
+      ["umap_0", "umap_1"],
+      categoricalSelection,
+      world.schema.annotations.obsByName
+    );
+
+    // Check to see that a centroid has been calculated for every categorical value
+    const keysAsArray = Array.from(centroidResult.keys());
+    expect(keysAsArray).toEqual(expect.arrayContaining([false, true]));
+
+    // This expected result assumes that all cells belong in all categorical values inside of sample response
+    const expectedResult = [
+      quantile([0.5], world.obsLayout.col("umap_0").asArray())[0],
+      quantile([0.5], world.obsLayout.col("umap_1").asArray())[0]
+    ];
+
+    centroidResult.forEach(coordinate => {
+      expect(coordinate).toEqual(expectedResult);
+    });
+  });
+});

client/__tests__/util/dataframe/dataframe.test.js

@@ -321,7 +321,10 @@ describe("dataframe factories", () => {
     test("KeyIndex", () => {
       const df = new Dataframe.Dataframe(
         [2, 2],
-        [["red", "blue"], [true, false]],
+        [
+          ["red", "blue"],
+          [true, false]
+        ],
         null,
         new Dataframe.KeyIndex(["colors", "bools"])
       );
@@ -341,7 +344,10 @@ describe("dataframe factories", () => {
     test("DenseInt32Index", () => {
       const df = new Dataframe.Dataframe(
         [2, 2],
-        [["red", "blue"], [true, false]],
+        [
+          ["red", "blue"],
+          [true, false]
+        ],
         null,
         new Dataframe.DenseInt32Index([74, 75])
       );
@@ -363,7 +369,10 @@ describe("dataframe factories", () => {
     test("DenseInt32Index promote", () => {
       const df = new Dataframe.Dataframe(
         [2, 2],
-        [["red", "blue"], [true, false]],
+        [
+          ["red", "blue"],
+          [true, false]
+        ],
         null,
         new Dataframe.DenseInt32Index([74, 75])
       );
@@ -385,7 +394,10 @@ describe("dataframe factories", () => {
     test("IdentityInt32Index with last", () => {
       const df = new Dataframe.Dataframe(
         [2, 2],
-        [["red", "blue"], [true, false]],
+        [
+          ["red", "blue"],
+          [true, false]
+        ],
         null,
         null
       );
@@ -407,7 +419,10 @@ describe("dataframe factories", () => {
     test("IdentityInt32Index promote", () => {
       const df = new Dataframe.Dataframe(
         [2, 2],
-        [["red", "blue"], [true, false]],
+        [
+          ["red", "blue"],
+          [true, false]
+        ],
         null,
         null
       );
@@ -461,7 +476,11 @@ describe("dataframe factories", () => {
       */
       const dfA = new Dataframe.Dataframe(
         [2, 3],
-        [["red", "blue"], [true, false], [1, 0]],
+        [
+          ["red", "blue"],
+          [true, false],
+          [1, 0]
+        ],
         null,
         new Dataframe.KeyIndex(["colors", "bools", "numbers"])
       );
@@ -481,6 +500,7 @@ describe("dataframe factories", () => {
 
     test("simple", () => {
       /* simple test that it works as expected in common case */
+      const dfEmpty = Dataframe.Dataframe.empty();
       const dfA = new Dataframe.Dataframe(
         [2, 1],
         [["red", "blue"]],
@@ -494,6 +514,22 @@ describe("dataframe factories", () => {
         new Dataframe.KeyIndex(["bools"])
       );
 
+      const dfLikeA = dfEmpty.withColsFrom(dfA);
+      expect(dfLikeA).toBeDefined();
+      expect(dfLikeA.dims).toEqual(dfA.dims);
+      expect(dfLikeA.colIndex.keys()).toEqual(dfA.colIndex.keys());
+      expect(dfLikeA.rowIndex).toEqual(dfA.rowIndex);
+      expect(dfLikeA.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(dfLikeA.icol(0).asArray()).toEqual(dfA.icol(0).asArray());
+
+      const dfAlsoLikeA = dfA.withColsFrom(dfEmpty);
+      expect(dfAlsoLikeA).toBeDefined();
+      expect(dfAlsoLikeA.dims).toEqual(dfA.dims);
+      expect(dfAlsoLikeA.colIndex.keys()).toEqual(dfA.colIndex.keys());
+      expect(dfAlsoLikeA.rowIndex).toEqual(dfA.rowIndex);
+      expect(dfAlsoLikeA.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(dfAlsoLikeA.icol(0).asArray()).toEqual(dfA.icol(0).asArray());
+
       const dfC = dfA.withColsFrom(dfB);
       expect(dfC).toBeDefined();
       expect(dfC.dims).toEqual([2, 2]);
@@ -503,13 +539,87 @@ describe("dataframe factories", () => {
       expect(dfC.col("colors").asArray()).toEqual(["red", "blue"]);
       expect(dfC.col("bools").asArray()).toEqual([true, false]);
     });
+
+    test("column picking", () => {
+      const dfEmpty = Dataframe.Dataframe.empty();
+      const dfA = new Dataframe.Dataframe(
+        [2, 1],
+        [["red", "blue"]],
+        null,
+        new Dataframe.KeyIndex(["colors"])
+      );
+      const dfB = new Dataframe.Dataframe(
+        [2, 3],
+        [
+          ["red", "blue"],
+          [true, false],
+          [1, 0]
+        ],
+        null,
+        new Dataframe.KeyIndex(["colors", "bools", "numbers"])
+      );
+
+      const dfX = dfEmpty.withColsFrom(dfB, ["colors", "bools"]);
+      expect(dfX).toBeDefined();
+      expect(dfX.dims).toEqual([2, 2]);
+      expect(dfX.colIndex.keys()).toEqual(["colors", "bools"]);
+      expect(dfX.rowIndex).toEqual(dfB.rowIndex);
+      expect(dfX.icol(0).asArray()).toEqual(dfB.icol(0).asArray());
+
+      const dfY = dfA.withColsFrom(dfB, ["numbers"]);
+      expect(dfY).toBeDefined();
+      expect(dfY.dims).toEqual([2, 2]);
+      expect(dfY.colIndex.keys()).toEqual(["colors", "numbers"]);
+      expect(dfY.rowIndex).toEqual(dfA.rowIndex);
+      expect(dfY.icol(0).asArray()).toEqual(dfA.icol(0).asArray());
+
+      const dfZ = dfA.withColsFrom(dfEmpty, []);
+      expect(dfZ).toBeDefined();
+      expect(dfZ.dims).toEqual(dfA.dims);
+      expect(dfZ.colIndex.keys()).toEqual(dfA.colIndex.keys());
+      expect(dfZ.rowIndex).toEqual(dfA.rowIndex);
+      expect(dfZ.icol(0).asArray()).toEqual(dfA.icol(0).asArray());
+
+      expect(() => dfA.withColsFrom(dfB, ["bools", "colors"])).toThrow();
+    });
+
+    test("column aliasing", () => {
+      const dfA = new Dataframe.Dataframe(
+        [2, 1],
+        [["red", "blue"]],
+        null,
+        new Dataframe.KeyIndex(["colors"])
+      );
+      const dfB = new Dataframe.Dataframe(
+        [2, 3],
+        [
+          ["red", "blue"],
+          [true, false],
+          [1, 0]
+        ],
+        null,
+        new Dataframe.KeyIndex(["colors", "bools", "numbers"])
+      );
+
+      const dfX = dfA.withColsFrom(dfB, { colors: "_colors", bools: "_bools" });
+      expect(dfX).toBeDefined();
+      expect(dfX.dims).toEqual([2, 3]);
+      expect(dfX.colIndex.keys()).toEqual(["colors", "_colors", "_bools"]);
+      expect(dfX.rowIndex).toEqual(dfA.rowIndex);
+      expect(dfX.icol(0).asArray()).toEqual(dfA.icol(0).asArray());
+      expect(dfX.col("_colors").asArray()).toBe(dfB.col("colors").asArray());
+    });
   });
 
   describe("dropCol", () => {
     test("KeyIndex", () => {
       const df = new Dataframe.Dataframe(
         [2, 3],
-        [["red", "blue"], [true, false], [1, 0]],
+        [
+          ["red", "blue"],
+          [true, false],
+          [1, 0]
+        ],
         null,
         new Dataframe.KeyIndex(["colors", "bools", "numbers"])
       );
@@ -528,7 +638,11 @@ describe("dataframe factories", () => {
     test("IdentityInt32Index drop first", () => {
       const df = new Dataframe.Dataframe(
         [2, 3],
-        [["red", "blue"], [true, false], [1, 0]],
+        [
+          ["red", "blue"],
+          [true, false],
+          [1, 0]
+        ],
         null,
         null
       );
@@ -548,7 +662,11 @@ describe("dataframe factories", () => {
     test("IdentityInt32Index drop last", () => {
       const df = new Dataframe.Dataframe(
         [2, 3],
-        [["red", "blue"], [true, false], [1, 0]],
+        [
+          ["red", "blue"],
+          [true, false],
+          [1, 0]
+        ],
         null,
         null
       );
@@ -568,7 +686,11 @@ describe("dataframe factories", () => {
     test("DenseInt32Index", () => {
       const df = new Dataframe.Dataframe(
         [2, 3],
-        [["red", "blue"], [true, false], [1, 0]],
+        [
+          ["red", "blue"],
+          [true, false],
+          [1, 0]
+        ],
         null,
         new Dataframe.DenseInt32Index([102, 101, 100])
       );
@@ -636,7 +758,10 @@ describe("dataframe factories", () => {
     test("renameCol", () => {
       const dfA = new Dataframe.Dataframe(
         [2, 2],
-        [[true, false], [1, 0]],
+        [
+          [true, false],
+          [1, 0]
+        ],
         null,
         new Dataframe.KeyIndex(["A", "B"])
       );
@@ -654,7 +779,10 @@ describe("dataframe col", () => {
   beforeEach(() => {
     df = new Dataframe.Dataframe(
       [2, 2],
-      [[true, false], [1, 0]],
+      [
+        [true, false],
+        [1, 0]
+      ],
       null,
       new Dataframe.KeyIndex(["A", "B"])
     );

client/__tests__/util/stateManager/controlsHelpers.test.js

@@ -0,0 +1,24 @@
+/*
+test controls helpers
+*/
+import { subsetAndResetGeneLists } from "../../../src/util/stateManager/controlsHelpers";
+import * as globals from "../../../src/globals";
+
+describe("controls helpers", () => {
+
+  test("subsetAndResetGeneLists", () => {
+    const geneList = [...Array(150).keys()].map(() =>
+      Math.random().toString(36).substring(2, 6) // random string of 4 characters
+    );
+    const state = {
+      userDefinedGenes: geneList.slice(0, 20),
+      diffexpGenes: geneList.slice(20),
+    };
+    const [newUserDefinedGenes, newDiffExpGenes] = subsetAndResetGeneLists(state);
+    expect(globals.maxUserDefinedGenes).toBeLessThan(globals.maxGenes);
+    expect(geneList.length).toBeGreaterThan(globals.maxGenes);
+    expect(newUserDefinedGenes).toHaveLength(globals.maxGenes);
+    expect(newUserDefinedGenes).toStrictEqual(geneList.slice(0, globals.maxGenes));
+    expect(newDiffExpGenes).toStrictEqual([]);
+  });
+});

client/__tests__/util/stateManager/universe.test.js

@@ -30,15 +30,39 @@ describe("createUniverseFromResponse", () => {
     create a universe from sample data nad validate its shape & contents
     */
     const { nObs, nVar } = REST.schema.schema.dataframe;
-    const universe = Universe.createUniverseFromResponse(
+    let universe = Universe.createUniverseFromResponse(
       REST.config,
-      REST.schema,
+      REST.schema
-      REST.annotationsObs,
+    );
-      REST.annotationsVar,
+    expect(universe).toBeDefined();
-      REST.layoutObs
+    expect(universe).toMatchObject(
+      expect.objectContaining({
+        nObs,
+        nVar,
+        schema: REST.schema.schema,
+        obsAnnotations: expect.any(Dataframe.Dataframe),
+        varAnnotations: expect.any(Dataframe.Dataframe),
+        obsLayout: expect.any(Dataframe.Dataframe),
+        varData: expect.any(Dataframe.Dataframe)
+      })
     );
 
-    expect(universe).toBeDefined();
+    universe = {
+      ...universe,
+      ...Universe.addObsAnnotations(
+        universe,
+        Universe.matrixFBSToDataframe(REST.annotationsObs)
+      ),
+      ...Universe.addVarAnnotations(
+        universe,
+        Universe.matrixFBSToDataframe(REST.annotationsVar)
+      ),
+      ...Universe.addObsLayout(
+        universe,
+        Universe.matrixFBSToDataframe(REST.layoutObs)
+      )
+    };
+
     expect(universe).toMatchObject(
       expect.objectContaining({
         nObs,

client/__tests__/util/stateManager/world.test.js

@@ -17,13 +17,27 @@ the default REST test response.
 const defaultBigBang = () => {
   /* create unverse, world, crossfilter and dimensionMap */
   /* create universe */
-  const universe = Universe.createUniverseFromResponse(
+  let universe = Universe.createUniverseFromResponse(
     _.cloneDeep(REST.config),
-    _.cloneDeep(REST.schema),
+    _.cloneDeep(REST.schema)
-    _.cloneDeep(REST.annotationsObs),
-    _.cloneDeep(REST.annotationsVar),
-    _.cloneDeep(REST.layoutObs)
   );
+
+  universe = {
+    ...universe,
+    ...Universe.addObsAnnotations(
+      universe,
+      Universe.matrixFBSToDataframe(REST.annotationsObs)
+    ),
+    ...Universe.addVarAnnotations(
+      universe,
+      Universe.matrixFBSToDataframe(REST.annotationsVar)
+    ),
+    ...Universe.addObsLayout(
+      universe,
+      Universe.matrixFBSToDataframe(REST.layoutObs)
+    )
+  };
+
   /* create world */
   const world = World.createWorldFromEntireUniverse(universe);
   /* create crossfilter */
@@ -45,9 +59,9 @@ describe("createWorldFromEntireUniverse", () => {
     const universe = Universe.createUniverseFromResponse(
       _.cloneDeep(REST.config),
       _.cloneDeep(REST.schema),
-      _.cloneDeep(REST.annotationsObs),
+      Universe.matrixFBSToDataframe(_.cloneDeep(REST.annotationsObs)),
-      _.cloneDeep(REST.annotationsVar),
+      Universe.matrixFBSToDataframe(_.cloneDeep(REST.annotationsVar)),
-      _.cloneDeep(REST.layoutObs)
+      Universe.matrixFBSToDataframe(_.cloneDeep(REST.layoutObs))
     );
     expect(universe).toBeDefined();
 

client/__tests__/util/typedCrossfilter/bitArray.test.js

@@ -183,3 +183,24 @@ describe("fillBySelection", () => {
     expect(arr).toEqual(truth);
   });
 });
+
+describe("wide bitarray", () => {
+  test.each([9, 30, 31, 32, 33, 54, 63, 64, 65, 127, 128, 129])(
+    "more than %d dimensions",
+    ndim => {
+      /* ensure we move across the uint boundary correctly */
+
+      const ba = new BitArray(defaultTestLength);
+      expect(ba).toBeDefined();
+
+      for (let i = 0; i < ndim; i += 1) {
+        expect(ba.allocDimension()).toEqual(i);
+      }
+      ba.freeDimension(0);
+      expect(ba.allocDimension()).toEqual(0);
+
+      ba.freeDimension(ndim - 1);
+      expect(ba.allocDimension()).toEqual(ndim - 1);
+    }
+  );
+});

client/configuration/webpack/webpack.config.dev.js

@@ -88,6 +88,9 @@ module.exports = {
     new webpack.NoEmitOnErrorsPlugin(),
     new webpack.DefinePlugin({
       __REACT_DEVTOOLS_GLOBAL_HOOK__: "({ isDisabled: true })"
+    }),
+    new webpack.DefinePlugin({
+      "process.env.CXG_SERVER_PORT": JSON.stringify(process.env.CXG_SERVER_PORT),
     })
   ]
 };

client/package.json

@@ -1,24 +1,15 @@
 {
   "name": "cellxgene",
-  "version": "0.14.1",
+  "version": "0.15.0",
   "license": "MIT",
   "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
   "repository": "https://github.com/chanzuckerberg/cellxgene",
   "scripts": {
-    "backend-dev": "python3.6 -m venv cellxgene && source cellxgene/bin/activate && yes | pip uninstall cellxgene || true && pip install -e .. && cellxgene launch ",
-    "backend-dev-anno-ontology": "python3.6 -m venv cellxgene && source cellxgene/bin/activate && yes | pip uninstall cellxgene || true && pip install -e .. && cellxgene launch --experimental-annotations --experimental-annotations-ontology",
-    "backend-dev-anno": "python3.6 -m venv cellxgene && source cellxgene/bin/activate && yes | pip uninstall cellxgene || true && pip install -e .. && cellxgene launch --experimental-annotations",
-    "build": "npm run clean && webpack --config configuration/webpack/webpack.config.prod.js",
     "clean": "rimraf build",
+    "build": "npm run clean && webpack --config configuration/webpack/webpack.config.prod.js",
     "dev": "npm run clean && webpack --config configuration/webpack/webpack.config.dev.js",
-    "e2e": "node node_modules/jest/bin/jest.js --verbose false --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
     "fmt": "eslint --fix src",
-    "lint": "eslint src",
+    "lint": "eslint src"
-    "smoke-test": "start-server-and-test start-server-for-test :5000 e2e",
-    "start": "node server/development.js",
-    "start-server-for-test": "cellxgene launch -p 5000 ../example-dataset/pbmc3k.h5ad",
-    "test": "node node_modules/jest/bin/jest.js",
-    "unit-test": "node node_modules/jest/bin/jest.js --testPathIgnorePatterns e2e"
   },
   "engineStrict": true,
   "engines": {
@@ -34,80 +25,79 @@
     "eslint-scope": "3.7.1"
   },
   "dependencies": {
-    "@blueprintjs/core": "^3.22.3",
+    "@blueprintjs/core": "^3.24.0",
-    "@blueprintjs/icons": "^3.13.0",
+    "@blueprintjs/icons": "^3.14.0",
-    "@blueprintjs/select": "^3.11.2",
+    "@blueprintjs/select": "^3.12.0",
     "client": "0.0.1",
     "d3": "^4.10.0",
     "d3-scale-chromatic": "^1.5.0",
     "flatbuffers": "^1.11.0",
     "fuzzysort": "^1.1.4",
-    "gl-mat4": "^1.1.4",
+    "gl-mat4": "^1.2.0",
-    "gl-matrix": "^3.1.0",
+    "gl-matrix": "^3.2.1",
     "gl-vec3": "^1.1.3",
     "is-number": "^7.0.0",
     "lodash": "^4.17.15",
     "memoize-one": "^5.1.1",
-    "react": "^16.12.0",
+    "react": "^16.13.0",
-    "react-autocomplete": "^1.7.2",
+    "react-dom": "^16.13.0",
-    "react-dom": "^16.12.0",
     "react-flip-toolkit": "7.0.6",
     "react-helmet": "^5.2.1",
-    "react-icons": "^3.8.0",
+    "react-icons": "^3.9.0",
-    "react-redux": "^7.1.3",
+    "react-redux": "^7.2.0",
     "redux": "^4.0.5",
-    "redux-thunk": "^2.2.0",
+    "redux-thunk": "^2.3.0",
-    "regl": "^1.3.13"
+    "regl": "^1.4.0"
   },
   "devDependencies": {
-    "@babel/core": "^7.7.7",
+    "@babel/core": "^7.8.7",
-    "@babel/plugin-proposal-class-properties": "^7.7.4",
+    "@babel/plugin-proposal-class-properties": "^7.8.3",
-    "@babel/plugin-proposal-decorators": "^7.7.4",
+    "@babel/plugin-proposal-decorators": "^7.8.3",
-    "@babel/plugin-proposal-export-namespace-from": "^7.7.4",
+    "@babel/plugin-proposal-export-namespace-from": "^7.8.3",
-    "@babel/plugin-proposal-function-bind": "^7.7.4",
+    "@babel/plugin-proposal-function-bind": "^7.8.3",
-    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.7.4",
+    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.8.3",
-    "@babel/plugin-proposal-optional-chaining": "^7.7.5",
+    "@babel/plugin-proposal-optional-chaining": "^7.8.3",
-    "@babel/plugin-transform-react-constant-elements": "^7.7.4",
+    "@babel/plugin-transform-react-constant-elements": "^7.8.3",
-    "@babel/plugin-transform-runtime": "^7.7.6",
+    "@babel/plugin-transform-runtime": "^7.8.3",
-    "@babel/preset-env": "^7.7.7",
+    "@babel/preset-env": "^7.8.7",
-    "@babel/preset-react": "^7.7.4",
+    "@babel/preset-react": "^7.8.3",
-    "@babel/register": "^7.7.7",
+    "@babel/register": "^7.8.6",
-    "@babel/runtime": "^7.7.7",
+    "@babel/runtime": "^7.8.7",
-    "babel-eslint": "^10.0.3",
+    "babel-eslint": "^10.1.0",
-    "babel-jest": "^24.9.0",
+    "babel-jest": "^25.1.0",
     "babel-loader": "^8.0.6",
     "babel-preset-modern-browsers": "^14.1.1",
     "chalk": "^3.0.0",
+    "codecov": "^3.6.5",
     "connect-history-api-fallback": "^1.6.0",
     "copy-webpack-plugin": "^5.1.1",
-    "css-loader": "^3.4.1",
+    "css-loader": "^3.4.2",
     "eslint": "^6.8.0",
     "eslint-config-airbnb": "^18.0.1",
-    "eslint-config-prettier": "^6.9.0",
+    "eslint-config-prettier": "^6.10.0",
     "eslint-loader": "^3.0.3",
     "eslint-plugin-filenames": "^1.3.2",
-    "eslint-plugin-import": "^2.19.1",
+    "eslint-plugin-import": "^2.20.1",
-    "eslint-plugin-jest": "^23.3.0",
+    "eslint-plugin-jest": "^23.8.2",
     "eslint-plugin-jsx-a11y": "^6.2.3",
-    "eslint-plugin-react": "^7.17.0",
+    "eslint-plugin-react": "^7.19.0",
-    "eslint-plugin-react-hooks": "^2.3.0",
+    "eslint-plugin-react-hooks": "^2.5.0",
     "express": "^4.17.1",
-    "file-loader": "^5.0.2",
+    "file-loader": "^5.1.0",
     "html-webpack-inline-source-plugin": "0.0.10",
     "html-webpack-plugin": "^3.2.0",
-    "jest": "^24.9.0",
+    "jest": "^25.1.0",
     "jest-puppeteer": "^4.4.0",
-    "json-loader": "^0.5.4",
+    "json-loader": "^0.5.7",
     "mini-css-extract-plugin": "^0.9.0",
-    "puppeteer": "^2.0.0",
+    "puppeteer": "^2.1.1",
-    "rimraf": "^3.0.0",
+    "rimraf": "^3.0.2",
-    "serve-favicon": "^2.3.0",
+    "serve-favicon": "^2.5.0",
-    "start-server-and-test": "^1.10.6",
+    "style-loader": "^1.1.3",
-    "style-loader": "^1.1.2",
+    "sw-precache-webpack-plugin": "^1.0.0",
-    "sw-precache-webpack-plugin": "^0.11.5",
     "url-loader": "^3.0.0",
-    "webpack": "^4.41.5",
+    "webpack": "^4.42.0",
-    "webpack-cli": "^3.3.10",
+    "webpack-cli": "^3.3.11",
     "webpack-dev-middleware": "^3.7.2"
   },
   "jest": {
@@ -117,7 +107,9 @@
     "testURL": "http://localhost/",
     "setupFiles": [
       "./__tests__/setupMissingGlobals.js"
-    ]
+    ],
+    "coverageDirectory": "./coverage/",
+    "collectCoverage": true
   },
   "babel": {
     "env": {

client/server/development.js

@@ -11,7 +11,7 @@ var utils = require("./utils");
 
 process.env.NODE_ENV = "development";
 
-var PORT = process.env.PORT || 3000;
+const CLIENT_PORT = process.env.CXG_CLIENT_PORT;
 
 // Set up compiler
 var compiler = webpack(config);
@@ -22,7 +22,7 @@ compiler.plugin("invalid", () => {
 });
 
 compiler.plugin("done", stats => {
-  utils.formatStats(stats, PORT);
+  utils.formatStats(stats, CLIENT_PORT);
 });
 
 // Launch server
@@ -43,7 +43,7 @@ app.get("*", (req, res) => {
   res.sendFile(path.resolve("index.html"));
 });
 
-app.listen(PORT, err => {
+app.listen(CLIENT_PORT, err => {
   if (err) {
     console.log(err);
     return;

client/src/actions/index.js

@@ -1,4 +1,3 @@
-// jshint esversion: 6
 import _ from "lodash";
 import * as globals from "../globals";
 import { Universe, MatrixFBS } from "../util/stateManager";
@@ -8,6 +7,90 @@ import {
   doBinaryRequest,
   dispatchNetworkErrorMessageToUser
 } from "../util/actionHelpers";
+import { requestReembed, reembedResetWorldToUniverse } from "./reembed";
+
+/*
+return promise to fetch the OBS annotations we need to load.  Omit anything
+we don't need.
+*/
+function obsAnnotationFetchAndLoad(dispatch, schema) {
+  const obsAnnotations = schema?.schema?.annotations?.obs ?? {};
+  const columns = obsAnnotations.columns ?? [];
+  const index = obsAnnotations.index ?? false;
+  return Promise.all(
+    columns
+      .filter(col => col.name !== index)
+      .map(col => {
+        const path = `annotations/obs?annotation-name=${encodeURIComponent(
+          col.name
+        )}`;
+        const url = `${globals.API.prefix}${globals.API.version}${path}`;
+        return doBinaryRequest(url);
+      })
+      .map(rqst => rqst.then(buffer => Universe.matrixFBSToDataframe(buffer)))
+      .map(resp =>
+        resp.then(df =>
+          dispatch({
+            type: "universe: column load success",
+            dim: "obsAnnotations",
+            dataframe: df
+          })
+        )
+      )
+  );
+}
+
+/*
+return promise fetching VAR annotations we need to load.  Only index is currently used.
+*/
+function varAnnotationFetchAndLoad(dispatch, schema) {
+  const varAnnotations = schema?.schema?.annotations?.var ?? {};
+  const index = varAnnotations.index ?? false;
+  const names = index ? [index] : [];
+  return Promise.all(
+    names
+      .map(name => {
+        const path = `annotations/var?annotation-name=${encodeURIComponent(
+          name
+        )}`;
+        const url = `${globals.API.prefix}${globals.API.version}${path}`;
+        return doBinaryRequest(url);
+      })
+      .map(rqst => rqst.then(buffer => Universe.matrixFBSToDataframe(buffer)))
+      .map(resp =>
+        resp.then(df =>
+          dispatch({
+            type: "universe: column load success",
+            dim: "varAnnotations",
+            dataframe: df
+          })
+        )
+      )
+  );
+}
+
+/*
+return promise fetching layout we need
+*/
+function layoutFetchAndLoad(dispatch) {
+  return Promise.all(
+    ["layout/obs"]
+      .map(path => {
+        const url = `${globals.API.prefix}${globals.API.version}${path}`;
+        return doBinaryRequest(url);
+      })
+      .map(rqst => rqst.then(buffer => Universe.matrixFBSToDataframe(buffer)))
+      .map(resp =>
+        resp.then(df =>
+          dispatch({
+            type: "universe: column load success",
+            dim: "obsLayout",
+            dataframe: df
+          })
+        )
+      )
+  );
+}
 
 /*
 Bootstrap application with the initial data loading.
@@ -31,34 +114,29 @@ const doInitialDataLoad = () =>
       /* set config defaults */
       const config = { ...globals.configDefaults, ...stepOneResults[0].config };
       const schema = stepOneResults[1];
-
+      const universe = Universe.createUniverseFromResponse(config, schema);
-      /*
+      dispatch({
-      Step 2 - dataframes, all binary.  NOTE: uses results of step 1.
+        type: "universe exists, but loading is still in progress",
-      */
+        universe
-      /* only load names for var annotations, if possible*/
+      });
-      const varIndexName = schema?.schema?.annotations?.var?.index;
-      const varAnnotationsQuery = varIndexName
-        ? `?annotation-name=${encodeURIComponent(varIndexName)}`
-        : "";
-      const varAnnotationsURL = `annotations/var${varAnnotationsQuery}`;
-      const requestBinary = ["annotations/obs", varAnnotationsURL, "layout/obs"]
-        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
-        .map(url => doBinaryRequest(url));
-      const stepTwoResults = await Promise.all(requestBinary);
-      const [obsAnno, varAnno, obsLayout] = [...stepTwoResults];
-
-      const universe = Universe.createUniverseFromResponse(
-        config,
-        schema,
-        obsAnno,
-        varAnno,
-        obsLayout
-      );
-
       dispatch({
         type: "configuration load complete",
         config
       });
+
+      /*
+      Step 2 - load the minimum stuff required to display.
+      */
+      await Promise.all([
+        layoutFetchAndLoad(dispatch),
+        varAnnotationFetchAndLoad(dispatch, schema)
+      ]);
+
+      /*
+      Step 3 - load everything else
+      */
+      await obsAnnotationFetchAndLoad(dispatch, schema);
+
       dispatch({
         type: "initial data load complete (universe exists)",
         universe
@@ -72,7 +150,7 @@ const doInitialDataLoad = () =>
 Set the view (world) to current selection.   Placeholder for an async action
 which also does re-layout.
 */
-const regraph = () => (dispatch, getState) => {
+const setWorldToSelection = () => (dispatch, getState) => {
   const { universe, world, crossfilter } = getState();
   dispatch({
     type: "set World to current selection",
@@ -90,7 +168,7 @@ const dispatchExpressionErrors = (dispatch, res) => {
 };
 
 /*
-Fetch expression vectors for each gene in genes.   This is NOT an action
+Fetch expression vectors for each gene in genes.  This is NOT an action
 function, but rather a helper to be called from an action helper that
 needs expression data.
 
@@ -308,37 +386,13 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
   }
 };
 
-const resetInterface = () => (dispatch, getState) => {
+const resetWorldToUniverse = () => (dispatch, getState) => {
   const { universe } = getState();
-
+  reembedResetWorldToUniverse(dispatch, getState);
-  dispatch({
-    type: "user reset start"
-  });
-  dispatch({
-    type: "clear all user defined genes"
-  });
-  dispatch({
-    type: "clear differential expression"
-  });
-  dispatch({
-    type: "reset colorscale"
-  });
-  dispatch({
-    type: "reset centroid labels"
-  });
-  dispatch({
-    type: "clear scatterplot"
-  });
   dispatch({
     type: "reset World to eq Universe",
     universe
   });
-  dispatch({
-    type: "increment graph render counter"
-  });
-  dispatch({
-    type: "user reset end"
-  });
 };
 
 const saveObsAnnotations = () => async (dispatch, getState) => {
@@ -395,11 +449,12 @@ const saveObsAnnotations = () => async (dispatch, getState) => {
 };
 
 export default {
-  regraph,
-  resetInterface,
-  requestSingleGeneExpressionCountsForColoringPOST,
-  requestDifferentialExpression,
-  requestUserDefinedGene,
   doInitialDataLoad,
-  saveObsAnnotations
+  requestDifferentialExpression,
+  requestSingleGeneExpressionCountsForColoringPOST,
+  requestUserDefinedGene,
+  requestReembed,
+  resetWorldToUniverse,
+  saveObsAnnotations,
+  setWorldToSelection
 };

client/src/actions/reembed.js

@@ -0,0 +1,113 @@
+import { API } from "../globals";
+import { Universe } from "../util/stateManager";
+import {
+  postNetworkErrorToast,
+  postAsyncSuccessToast,
+  postAsyncFailureToast
+} from "../components/framework/toasters";
+
+function abortableFetch(request, opts, timeout = 0) {
+  const controller = new AbortController();
+  const { signal } = controller;
+
+  return {
+    abort: () => controller.abort(),
+    isAborted: () => signal.aborted,
+    ready: () => {
+      if (timeout) {
+        setTimeout(() => controller.abort(), timeout);
+      }
+      return fetch(request, { ...opts, signal });
+    }
+  };
+}
+
+async function doReembedFetch(dispatch, getState) {
+  const state = getState();
+  let cells = state.world.obsAnnotations.rowIndex.keys();
+
+  // These lines ensure that we convert any TypedArray to an Array.
+  // This is necessary because JSON.stringify() does some very strange
+  // things with TypedArrays (they are marshalled to JSON objects, rather
+  // than being marshalled as a JSON array).
+  cells = Array.isArray(cells) ? cells : Array.from(cells);
+
+  const af = abortableFetch(
+    `${API.prefix}${API.version}layout/obs`,
+    {
+      method: "PUT",
+      headers: new Headers({
+        Accept: "application/octet-stream",
+        "Content-Type": "application/json"
+      }),
+      body: JSON.stringify({
+        method: "umap",
+        filter: { obs: { index: cells } }
+      }),
+      credentials: "include"
+    },
+    60000 // 1 minute timeout
+  );
+  dispatch({
+    type: "reembed: request start",
+    abortableFetch: af
+  });
+  const res = await af.ready();
+
+  if (
+    res.ok &&
+    res.headers.get("Content-Type").includes("application/octet-stream")
+  ) {
+    return res;
+  }
+
+  // else an error
+  let msg = `Unexpected HTTP response ${res.status}, ${res.statusText}`;
+  const body = await res.text();
+  if (body && body.length > 0) {
+    msg = `${msg} -- ${body}`;
+  }
+  postNetworkErrorToast(msg);
+  throw new Error(msg);
+}
+
+/*
+functions below are dispatch-able
+*/
+export function requestReembed() {
+  return async (dispatch, getState) => {
+    try {
+      const res = await doReembedFetch(dispatch, getState);
+      const schema = JSON.parse(res.headers.get("CxG-Schema"));
+      const buffer = await res.arrayBuffer();
+      const df = Universe.matrixFBSToDataframe(buffer);
+      dispatch({
+        type: "reembed: request completed"
+      });
+      dispatch({
+        type: "reembed: add reembedding",
+        embedding: df,
+        schema
+      });
+      postAsyncSuccessToast("Re-embedding has completed.");
+    } catch (error) {
+      dispatch({
+        type: "reembed: request aborted"
+      });
+      if (error.name === "AbortError") {
+        postAsyncFailureToast("Re-embedding calculation was aborted.");
+      } else {
+        postNetworkErrorToast(`Re-embedding: ${error.message}`);
+      }
+      console.log("Reembed exception:", error, error.name, error.message);
+    }
+  };
+}
+
+export function reembedResetWorldToUniverse(dispatch, getState) {
+  const { reembedController } = getState();
+  if (reembedController.pendingFetch) reembedController.pendingFetch.abort();
+  dispatch({
+    type: "reembed: clear all reembeddings"
+  });
+}

client/src/components/autosave/filenameDialog.js

@@ -74,7 +74,7 @@ class FilenameDialog extends React.Component {
             color: Colors.ORANGE3
           }}
         >
-          {"Name cannot be blank"}
+          Name cannot be blank
         </span>
       );
     } else if (err === "characters") {
@@ -87,7 +87,7 @@ class FilenameDialog extends React.Component {
             color: Colors.ORANGE3
           }}
         >
-          {"Only alphanumeric and underscore allowed"}
+          Only alphanumeric and underscore allowed
         </span>
       );
     }

client/src/components/autosave/index.js

@@ -11,7 +11,8 @@ import FilenameDialog from "./filenameDialog";
   saveInProgress: state.autosave?.saveInProgress ?? false,
   lastSavedObsAnnotations: state.autosave?.lastSavedObsAnnotations,
   error: state.autosave?.error,
-  writableCategoriesEnabled: state.config?.parameters?.["annotations"] ?? false
+  writableCategoriesEnabled: state.config?.parameters?.["annotations"] ?? false,
+  initialDataLoadComplete: state.autosave?.initialDataLoadComplete
 }))
 class Autosave extends React.Component {
   constructor(props) {
@@ -63,10 +64,17 @@ class Autosave extends React.Component {
   }
 
   render() {
-    const { writableCategoriesEnabled } = this.props;
+    const { writableCategoriesEnabled, saveInProgress, initialDataLoadComplete } = this.props;
     return writableCategoriesEnabled ? (
       <div
         id="autosave"
+        data-testclass={
+          !initialDataLoadComplete
+            ? "autosave-init"
+            : (this.needToSave() || saveInProgress)
+              ? "autosave-incomplete"
+              : "autosave-complete"
+        }
         style={{
           position: "fixed",
           display: "inherit",

client/src/components/brushableHistogram/index.js

@@ -351,8 +351,8 @@ class HistogramBrush extends React.PureComponent {
     const brushX = d3
       .brushX()
       .extent([
-          [x.range()[0], y.range()[1]],
+        [x.range()[0], y.range()[1]],
-          [x.range()[1], this.marginTop + this.height + this.marginBottom]
+        [x.range()[1], this.marginTop + this.height + this.marginBottom]
       ])
       /*
       emit start so that the Undoable history can save an undo point

client/src/components/categorical/annoDialog.js

@@ -6,11 +6,9 @@ import { Button, Tooltip, Dialog, Classes, Colors } from "@blueprintjs/core";
   colorAccessor: state.colors.colorAccessor,
   categoricalSelection: state.categoricalSelection,
   annotations: state.annotations,
-  universe: state.universe,
+  universe: state.universe
-  ontology: state.ontology,
-  ontologyLoading: state.ontology?.loading
 }))
-class AnnoDialog extends React.Component {
+class AnnoDialog extends React.PureComponent {
   constructor(props) {
     super(props);
     this.state = {};
@@ -26,13 +24,13 @@ class AnnoDialog extends React.Component {
       errorMessage,
       validationError,
       annoSelect,
-      ontologySelect,
       annoInput,
       handleCancel,
       handleSubmit,
       primaryButtonText,
       secondaryButtonText,
-      handleSecondaryButtonSubmit
+      handleSecondaryButtonSubmit,
+      primaryButtonProps
     } = this.props;
 
     return (
@@ -57,7 +55,6 @@ class AnnoDialog extends React.Component {
               </p>
             </div>
             {annoSelect || null}
-            {ontologySelect || null}
           </div>
           <div className={Classes.DIALOG_FOOTER}>
             <div className={Classes.DIALOG_FOOTER_ACTIONS}>
@@ -69,12 +66,13 @@ class AnnoDialog extends React.Component {
                   onClick={handleSecondaryButtonSubmit}
                   disabled={!text || validationError}
                   intent="none"
-                  type="submit"
+                  type="button"
                 >
                   {secondaryButtonText}
                 </Button>
               ) : null}
               <Button
+                {...primaryButtonProps} // eslint-disable-line react/jsx-props-no-spreading
                 onClick={handleSubmit}
                 disabled={!text || validationError}
                 intent="primary"

client/src/components/categorical/annoDialogAddLabel.js

@@ -1,9 +1,8 @@
 import React from "react";
 import { connect } from "react-redux";
 import AnnoDialog from "./annoDialog";
-import AnnoInputs from "./annoInputs";
+import LabelInput from "./labelInput";
-
+import { labelPrompt, isLabelErroneous } from "./labelUtil";
-import { labelErrorMessage, isLabelErroneous } from "./labelUtil";
 
 @connect(state => ({
   colorAccessor: state.colors.colorAccessor,
@@ -11,10 +10,9 @@ import { labelErrorMessage, isLabelErroneous } from "./labelUtil";
   annotations: state.annotations,
   universe: state.universe,
   ontology: state.ontology,
-  ontologyLoading: state.ontology?.loading,
+  crossfilter: state.crossfilter
-  ontologyEnabled: state.ontology?.enabled
 }))
-class Category extends React.Component {
+class Category extends React.PureComponent {
   constructor(props) {
     super(props);
     this.state = {
@@ -22,53 +20,43 @@ class Category extends React.Component {
     };
   }
 
-  disableAddNewLabelMode = () => {
+  disableAddNewLabelMode = e => {
     const { dispatch } = this.props;
-    dispatch({
-      type: "annotation: disable add new label mode"
-    });
     this.setState({
       newLabelText: ""
     });
+    dispatch({
+      type: "annotation: disable add new label mode"
+    });
+    if (e) e.preventDefault();
   };
 
-  handleAddNewLabelToCategory = () => {
+  handleAddNewLabelToCategory = e => {
     const { dispatch, metadataField } = this.props;
     const { newLabelText } = this.state;
 
+    this.disableAddNewLabelMode();
     dispatch({
       type: "annotation: add new label to category",
       metadataField,
       newLabelText,
       assignSelectedCells: false
     });
-    this.setState({ newLabelText: "" });
+    e.preventDefault();
   };
 
-  addLabelAndAssignCells = () => {
+  addLabelAndAssignCells = e => {
     const { dispatch, metadataField } = this.props;
     const { newLabelText } = this.state;
 
+    this.disableAddNewLabelMode();
     dispatch({
       type: "annotation: add new label to category",
       metadataField,
       newLabelText,
       assignSelectedCells: true
     });
-
+    e.preventDefault();
-    this.setState({ newLabelText: "" });
-  };
-
-  handleCreateArbitraryLabel = newLabelTextNotInOntology => {
-    const { dispatch, metadataField } = this.props;
-
-    dispatch({
-      type: "annotation: add new label to category",
-      metadataField,
-      newLabelText: newLabelTextNotInOntology,
-      assignSelectedCells: false
-    });
-    this.setState({ newLabelText: "" });
   };
 
   labelNameError = name => {
@@ -76,23 +64,18 @@ class Category extends React.Component {
     return isLabelErroneous(name, metadataField, ontology, universe.schema);
   };
 
-  labelNameErrorMessage = name => {
+  instruction = label => {
-    const { metadataField, ontology, universe } = this.props;
+    return labelPrompt(this.labelNameError(label), "New, unique label", ":");
-    return labelErrorMessage(name, metadataField, ontology, universe.schema);
   };
 
-  /* leaky to have both of these in multiple components */
+  handleChangeOrSelect = label => {
-  handleChoice = e => {
+    this.setState({ newLabelText: label });
-    this.setState({ newLabelText: e.target });
-  };
-
-  handleTextChange = text => {
-    this.setState({ newLabelText: text });
   };
 
   render() {
     const { newLabelText } = this.state;
-    const { metadataField, annotations, ontologyEnabled } = this.props;
+    const { metadataField, annotations, ontology, crossfilter } = this.props;
+    const ontologyEnabled = ontology?.enabled ?? false;
 
     return (
       <>
@@ -101,27 +84,31 @@ class Category extends React.Component {
             annotations.isAddingNewLabel &&
             annotations.categoryAddingNewLabel === metadataField
           }
+          inputProps={{ "data-testid": `${metadataField}:create-label-dialog` }}
+          primaryButtonProps={{
+            "data-testid": `${metadataField}:submit-label`
+          }}
           title="Add new label to category"
-          instruction="New, unique label name:"
+          instruction={this.instruction(newLabelText)}
           cancelTooltipContent="Close this dialog without adding a label."
           primaryButtonText="Add label"
-          secondaryButtonText="Add label and assign currently selected cells"
+          secondaryButtonText={`Add label and assign ${crossfilter.countSelected()} currently selected cells`}
           handleSecondaryButtonSubmit={this.addLabelAndAssignCells}
           text={newLabelText}
           validationError={this.labelNameError(newLabelText)}
-          errorMessage={this.labelNameErrorMessage(newLabelText)}
           handleSubmit={this.handleAddNewLabelToCategory}
           handleCancel={this.disableAddNewLabelMode}
           annoInput={
-            <AnnoInputs
+            <LabelInput
-              useSuggest={ontologyEnabled}
+              labelSuggestions={ontologyEnabled ? ontology.terms : null}
-              text={newLabelText}
+              onChange={this.handleChangeOrSelect}
-              handleCreateArbitraryLabel={this.handleCreateArbitraryLabel}
+              onSelect={this.handleChangeOrSelect}
-              handleItemChange={this.handleSuggestActiveItemChange}
+              inputProps={{
-              handleChoice={this.handleChoice}
+                "data-testid": `${metadataField}:new-label-name`,
-              handleTextChange={this.handleTextChange}
+                leftIcon: "tag",
-              isTextInvalid={this.labelNameError}
+                intent: "none",
-              isTextInvalidErrorMessage={this.labelNameErrorMessage}
+                autoFocus: true
+              }}
             />
           }
         />

client/src/components/categorical/annoDialogAddLabelFromOntology.js

@@ -1,131 +0,0 @@
-import React from "react";
-import { connect } from "react-redux";
-import AnnoDialog from "./annoDialog";
-import OntologySelect from "./ontologySelect";
-
-import { labelErrorMessage, isLabelErroneous } from "./labelUtil";
-
-@connect(state => ({
-  colorAccessor: state.colors.colorAccessor,
-  categoricalSelection: state.categoricalSelection,
-  annotations: state.annotations,
-  universe: state.universe,
-  ontology: state.ontology,
-  ontologyLoading: state.ontology?.loading,
-  ontologyEnabled: state.ontology?.enabled
-}))
-class Category extends React.Component {
-  constructor(props) {
-    super(props);
-    this.state = {
-      newLabelText: ""
-    };
-  }
-
-  disableAddNewLabelFromOntologyMode = () => {
-    const { dispatch } = this.props;
-    dispatch({
-      type: "annotation: disable add new ontology label mode"
-    });
-    this.setState({
-      newLabelText: ""
-    });
-  };
-
-  handleAddNewLabelToCategory = () => {
-    const { dispatch, metadataField } = this.props;
-    const { newLabelText } = this.state;
-
-    dispatch({
-      type: "annotation: add new label to category",
-      metadataField,
-      newLabelText,
-      assignSelectedCells: false
-    });
-    this.setState({ newLabelText: "" });
-  };
-
-  addLabelAndAssignCells = () => {
-    const { dispatch, metadataField } = this.props;
-    const { newLabelText } = this.state;
-
-    dispatch({
-      type: "annotation: add new label to category",
-      metadataField,
-      newLabelText,
-      assignSelectedCells: true
-    });
-
-    this.setState({ newLabelText: "" });
-  };
-
-  handleCreateArbitraryLabel = newLabelTextNotInOntology => {
-    const { dispatch, metadataField } = this.props;
-
-    dispatch({
-      type: "annotation: add new label to category",
-      metadataField,
-      newLabelText: newLabelTextNotInOntology,
-      assignSelectedCells: false
-    });
-    this.setState({ newLabelText: "" });
-  };
-
-  labelNameError = name => {
-    const { metadataField, ontology, universe } = this.props;
-    return isLabelErroneous(name, metadataField, ontology, universe.schema);
-  };
-
-  labelNameErrorMessage = name => {
-    const { metadataField, ontology, universe } = this.props;
-    return labelErrorMessage(name, metadataField, ontology, universe.schema);
-  };
-
-  /* leaky to have both of these in multiple components */
-  handleChoice = e => {
-    this.setState({ newLabelText: e.target });
-  };
-
-  handleTextChange = text => {
-    this.setState({ newLabelText: text });
-  };
-
-  render() {
-    const { newLabelText } = this.state;
-    const { metadataField, annotations, ontology } = this.props;
-
-    return (
-      <>
-        <AnnoDialog
-          isActive={
-            annotations.isAddingNewLabelFromOntology &&
-            annotations.categoryAddingNewLabelFromOntology === metadataField
-          }
-          title="Add new label to category from existing ontology terms"
-          instruction="Choose an ontology term to use as a label name:"
-          cancelTooltipContent="Close this dialog without adding a label."
-          primaryButtonText="Add label"
-          secondaryButtonText="Add label and assign currently selected cells"
-          handleSecondaryButtonSubmit={this.addLabelAndAssignCells}
-          text={newLabelText}
-          validationError={this.labelNameError(newLabelText)}
-          errorMessage={this.labelNameErrorMessage(newLabelText)}
-          handleSubmit={this.handleAddNewLabelToCategory}
-          handleCancel={this.disableAddNewLabelFromOntologyMode}
-          ontologySelect={
-            <OntologySelect
-              handleChooseOntologyTermFromDropdown={term => {
-                this.handleChoice(term);
-              }}
-              categoryToDuplicate={newLabelText}
-              ontology={ontology}
-            />
-          }
-          annoInput={null}
-        />
-      </>
-    );
-  }
-}
-
-export default Category;

client/src/components/categorical/annoDialogEditCategoryName.js

@@ -1,9 +1,9 @@
 import React from "react";
 import _ from "lodash";
 import { connect } from "react-redux";
-import { Colors } from "@blueprintjs/core";
 import AnnoDialog from "./annoDialog";
-import AnnoInputs from "./annoInputs";
+import LabelInput from "./labelInput";
+import { labelPrompt } from "./labelUtil";
 
 import { AnnotationsHelpers } from "../../util/stateManager";
 
@@ -11,11 +11,9 @@ import { AnnotationsHelpers } from "../../util/stateManager";
   categoricalSelection: state.categoricalSelection,
   annotations: state.annotations,
   universe: state.universe,
-  ontology: state.ontology,
+  ontology: state.ontology
-  ontologyLoading: state.ontology?.loading,
-  ontologyEnabled: state.ontology?.enabled
 }))
-class AnnoDialogEditCategoryName extends React.Component {
+class AnnoDialogEditCategoryName extends React.PureComponent {
   constructor(props) {
     super(props);
     this.state = {
@@ -23,20 +21,21 @@ class AnnoDialogEditCategoryName extends React.Component {
     };
   }
 
-  handleCategoryEditTextChange = txt => {
+  handleChangeOrSelect = name => {
     this.setState({
-      newCategoryText: txt
+      newCategoryText: name
     });
   };
 
   disableEditCategoryMode = () => {
-    const { dispatch } = this.props;
+    const { dispatch, metadataField } = this.props;
     dispatch({
       type: "annotation: disable category edit mode"
     });
+    this.setState({ newCategoryText: metadataField });
   };
 
-  handleEditCategory = () => {
+  handleEditCategory = e => {
     const { dispatch, metadataField, categoricalSelection } = this.props;
     const { newCategoryText } = this.state;
 
@@ -50,58 +49,29 @@ class AnnoDialogEditCategoryName extends React.Component {
       return;
     }
 
+    this.disableEditCategoryMode();
     dispatch({
       type: "annotation: category edited",
       metadataField,
       newCategoryText,
       data: newCategoryText
     });
+    e.preventDefault();
   };
 
-  categoryNameErrorMessage = () => {
+  editedCategoryNameError = name => {
-    const err = this.editedCategoryNameError();
-    if (err === false) return null;
-
-    const errorMessageMap = {
-      /* map error code to human readable error message */
-      "empty-string": "Blank names not allowed",
-      duplicate: "Category name must be unique",
-      "trim-spaces": "Leading and trailing spaces not allowed",
-      "illegal-characters":
-        "Only alphanumeric and special characters (-_.) allowed",
-      "multi-space-run": "Multiple consecutive spaces not allowed"
-    };
-    const errorMessage = errorMessageMap[err] ?? "error";
-    return (
-      <span
-        style={{
-          display: "block",
-          fontStyle: "italic",
-          fontSize: 12,
-          marginTop: 5,
-          color: Colors.ORANGE3
-        }}
-      >
-        {errorMessage}
-      </span>
-    );
-  };
-
-  editedCategoryNameError = () => {
     const { metadataField, categoricalSelection } = this.props;
-    const { newCategoryText } = this.state;
 
     /* check for syntax errors in category name */
-    const error = AnnotationsHelpers.annotationNameIsErroneous(newCategoryText);
+    const error = AnnotationsHelpers.annotationNameIsErroneous(name);
     if (error) {
       return error;
     }
 
     /* check for duplicative categories */
     const allCategoryNames = _.keys(categoricalSelection);
-    const categoryNameAlreadyExists =
+    const categoryNameAlreadyExists = allCategoryNames.indexOf(name) > -1;
-      allCategoryNames.indexOf(newCategoryText) > -1;
+    const sameName = name === metadataField;
-    const sameName = newCategoryText === metadataField;
     if (categoryNameAlreadyExists && !sameName) {
       return "duplicate";
     }
@@ -110,9 +80,18 @@ class AnnoDialogEditCategoryName extends React.Component {
     return false;
   };
 
+  instruction = name => {
+    return labelPrompt(
+      this.editedCategoryNameError(name),
+      "New, unique category name",
+      ":"
+    );
+  };
+
   render() {
     const { newCategoryText } = this.state;
-    const { metadataField, annotations } = this.props;
+    const { metadataField, annotations, ontology } = this.props;
+    const ontologyEnabled = ontology?.enabled ?? false;
 
     return (
       <>
@@ -121,20 +100,33 @@ class AnnoDialogEditCategoryName extends React.Component {
             annotations.isEditingCategoryName &&
             annotations.categoryBeingEdited === metadataField
           }
+          inputProps={{
+            "data-testid": `${metadataField}:edit-category-name-dialog`
+          }}
+          primaryButtonProps={{
+            "data-testid": `${metadataField}:submit-category-edit`
+          }}
           title="Edit category name"
-          instruction="New, unique category name:"
+          instruction={this.instruction(newCategoryText)}
           cancelTooltipContent="Close this dialog without editing this category."
           primaryButtonText="Edit category name"
           text={newCategoryText}
           validationError={this.editedCategoryNameError(newCategoryText)}
-          errorMessage={this.categoryNameErrorMessage(newCategoryText)}
           handleSubmit={this.handleEditCategory}
           handleCancel={this.disableEditCategoryMode}
           annoInput={
-            <AnnoInputs
+            <LabelInput
-              useSuggest={false}
+              label={newCategoryText}
-              text={newCategoryText}
+              labelSuggestions={ontologyEnabled ? ontology.terms : null}
-              handleTextChange={this.handleCategoryEditTextChange}
+              onChange={this.handleChangeOrSelect}
+              onSelect={this.handleChangeOrSelect}
+              inputProps={{
+                "data-testid": `${metadataField}:edit-category-name-text`,
+                leftIcon: "tag",
+                intent: "none",
+                autoFocus: true
+              }}
+              newLabelMessage="New category"
             />
           }
         />

client/src/components/categorical/annoInputs.js

@@ -1,43 +0,0 @@
-import React from "react";
-import { connect } from "react-redux";
-import { InputGroup } from "@blueprintjs/core";
-
-const VanillaInput = props => {
-  const { text, handleTextChange } = props;
-  return (
-    <InputGroup
-      autoFocus
-      value={text}
-      intent="none"
-      onChange={e => handleTextChange?.(e.target.value)}
-      leftIcon="tag"
-    />
-  );
-};
-
-@connect(state => ({
-  colorAccessor: state.colors.colorAccessor,
-  categoricalSelection: state.categoricalSelection,
-  annotations: state.annotations,
-  universe: state.universe,
-  world: state.world,
-  ontology: state.ontology?.terms,
-  ontologyLoading: state.ontology?.loading
-}))
-class AnnoInputs extends React.Component {
-  constructor(props) {
-    super(props);
-    this.state = {};
-  }
-
-  render() {
-    const { handleTextChange, text } = this.props;
-    return (
-      <div>
-        <VanillaInput text={text} handleTextChange={handleTextChange} />
-      </div>
-    );
-  }
-}
-
-export default AnnoInputs;

client/src/components/categorical/annoMenuCategory.js

@@ -10,10 +10,9 @@ import {
 } from "@blueprintjs/core";
 
 @connect(state => ({
-  annotations: state.annotations,
+  annotations: state.annotations
-  ontologyEnabled: state.ontology?.enabled
 }))
-class AnnoMenuCategory extends React.Component {
+class AnnoMenuCategory extends React.PureComponent {
   constructor(props) {
     super(props);
     this.state = {};
@@ -27,14 +26,6 @@ class AnnoMenuCategory extends React.Component {
     });
   };
 
-  activateAddNewOntologyLabelMode = () => {
-    const { dispatch, metadataField } = this.props;
-    dispatch({
-      type: "annotation: activate add new ontology label mode",
-      data: metadataField
-    });
-  };
-
   activateEditCategoryMode = () => {
     const { dispatch, metadataField } = this.props;
 
@@ -57,9 +48,7 @@ class AnnoMenuCategory extends React.Component {
       metadataField,
       annotations,
       isUserAnno,
-      ontologyEnabled,
       createText,
-      createFromOntologyText,
       editText,
       deleteText
     } = this.props;
@@ -76,24 +65,15 @@ class AnnoMenuCategory extends React.Component {
                 <MenuItem
                   icon="tag"
                   data-testclass="handleAddNewLabelToCategory"
-                  data-testid={`handleAddNewLabelToCategory-${metadataField}`}
+                  data-testid={`${metadataField}:add-new-label-to-category`}
                   onClick={this.activateAddNewLabelMode}
                   text={createText}
                 />
-                {ontologyEnabled ? (
-                  <MenuItem
-                    icon="book"
-                    data-testclass="activateAddNewOntologyLabelMode"
-                    data-testid={`activateAddNewOntologyLabelMode-${metadataField}`}
-                    onClick={this.activateAddNewOntologyLabelMode}
-                    text={createFromOntologyText}
-                  />
-                ) : null}
                 <MenuItem
                   icon="edit"
                   disabled={annotations.isEditingCategoryName}
                   data-testclass="activateEditCategoryMode"
-                  data-testid={`activateEditCategoryMode-${metadataField}`}
+                  data-testid={`${metadataField}:edit-category-mode`}
                   onClick={this.activateEditCategoryMode}
                   text={editText}
                 />
@@ -101,7 +81,7 @@ class AnnoMenuCategory extends React.Component {
                   icon="delete"
                   intent="danger"
                   data-testclass="handleDeleteCategory"
-                  data-testid={`handleDeleteCategory-${metadataField}`}
+                  data-testid={`${metadataField}:delete-category`}
                   onClick={this.handleDeleteCategory}
                   text={deleteText}
                 />
@@ -111,7 +91,7 @@ class AnnoMenuCategory extends React.Component {
             <Button
               style={{ marginLeft: 0 }}
               data-testclass="seeActions"
-              data-testid={`seeActions-${metadataField}`}
+              data-testid={`${metadataField}:see-actions`}
               icon="more"
               minimal
             />

client/src/components/categorical/annoSelect.js

@@ -7,11 +7,9 @@ import { Select } from "@blueprintjs/select";
   colorAccessor: state.colors.colorAccessor,
   categoricalSelection: state.categoricalSelection,
   annotations: state.annotations,
-  universe: state.universe,
+  universe: state.universe
-  ontology: state.ontology,
-  ontologyLoading: state.ontology?.loading
 }))
-class DuplicateCategorySelect extends React.Component {
+class DuplicateCategorySelect extends React.PureComponent {
   constructor(props) {
     super(props);
     this.state = {};

client/src/components/categorical/categorical.js

@@ -4,15 +4,17 @@ import { Button } from "@blueprintjs/core";
 import { connect } from "react-redux";
 import * as globals from "../../globals";
 import Category from "./category";
-import { AnnotationsHelpers } from "../../util/stateManager";
+import { AnnotationsHelpers, ControlsHelpers } from "../../util/stateManager";
 import AnnoDialog from "./annoDialog";
-import AnnoInputs from "./annoInputs";
 import AnnoSelect from "./annoSelect";
+import LabelInput from "./labelInput";
+import { labelPrompt } from "./labelUtil";
 
 @connect(state => ({
-  categoricalSelection: state.categoricalSelection,
   writableCategoriesEnabled: state.config?.parameters?.["annotations"] ?? false,
-  schema: state.world?.schema
+  schema: state.world?.schema,
+  config: state.config,
+  ontology: state.ontology
 }))
 class Categories extends React.Component {
   constructor(props) {
@@ -20,11 +22,12 @@ class Categories extends React.Component {
     this.state = {
       createAnnoModeActive: false,
       newCategoryText: "",
-      categoryToDuplicate: null
+      categoryToDuplicate: null,
+      expandedCats: new Set()
     };
   }
 
-  handleCreateUserAnno = () => {
+  handleCreateUserAnno = e => {
     const { dispatch } = this.props;
     const { newCategoryText, categoryToDuplicate } = this.state;
     dispatch({
@@ -37,6 +40,7 @@ class Categories extends React.Component {
       categoryToDuplicate: null,
       newCategoryText: ""
     });
+    e.preventDefault();
   };
 
   handleEnableAnnoMode = () => {
@@ -86,49 +90,49 @@ class Categories extends React.Component {
     return false;
   };
 
-  categoryNameErrorMessage = name => {
+  handleChange = name => {
-    const err = this.categoryNameError(name);
+    this.setState({ newCategoryText: name });
-    if (err === false) return null;
-
-    const errorMessageMap = {
-      /* map error code to human readable error message */
-      "empty-string": "Blank names not allowed",
-      duplicate: "Name must be unique",
-      "trim-spaces": "Leading and trailing spaces not allowed",
-      "illegal-characters":
-        "Only alphanumeric and special characters (-_.) allowed",
-      "multi-space-run": "Multiple consecutive spaces not allowed"
-    };
-    const errorMessage = errorMessageMap[err] ?? "error";
-    return <span>{errorMessage}</span>;
   };
 
-  handleNewCategoryText = txt => {
+  handleSelect = name => {
-    this.setState({ newCategoryText: txt });
+    this.setState({ newCategoryText: name });
   };
 
-  handleChoice = e => {
+  instruction = name => {
-    /* Blueprint Suggest format */
+    return labelPrompt(
-    this.setState({ newCategoryText: e.target });
+      this.categoryNameError(name),
+      "New, unique category name",
+      ":"
+    );
   };
 
-  handleSuggestActiveItemChange = () => {};
+  onExpansionChange = catName => {
+    const {expandedCats} = this.state;
+    if (expandedCats.has(catName)) {
+      const _expandedCats = new Set(expandedCats);
+      _expandedCats.delete(catName)
+      this.setState({expandedCats: _expandedCats}) 
+    } else {
+      const _expandedCats = new Set(expandedCats);
+      _expandedCats.add(catName)
+      this.setState({expandedCats: _expandedCats})
+    }
+  }
 
   render() {
     const {
       createAnnoModeActive,
       categoryToDuplicate,
-      newCategoryText
+      newCategoryText,
+      expandedCats      
     } = this.state;
-    const {
+    const { writableCategoriesEnabled, schema, config, ontology } = this.props;
-      categoricalSelection,
+    const ontologyEnabled = ontology?.enabled ?? false;
-      writableCategoriesEnabled,
-      schema
-    } = this.props;
-    if (!categoricalSelection) return null;
-
     /* all names, sorted in display order.  Will be rendered in this order */
-    const allCategoryNames = Object.keys(categoricalSelection).sort();
+    const allCategoryNames = ControlsHelpers.selectableCategoryNames(
+      schema,
+      ControlsHelpers.maxCategoryItems(config)
+    ).sort();
 
     return (
       <div
@@ -139,20 +143,26 @@ class Categories extends React.Component {
         <AnnoDialog
           isActive={createAnnoModeActive}
           title="Create new category"
-          instruction="New, unique category name:"
+          instruction={this.instruction(newCategoryText)}
           cancelTooltipContent="Close this dialog without creating a category."
           primaryButtonText="Create new category"
+          primaryButtonProps={{ "data-testid": "submit-category" }}
           text={newCategoryText}
           validationError={this.categoryNameError(newCategoryText)}
-          errorMessage={this.categoryNameErrorMessage(newCategoryText)}
           handleSubmit={this.handleCreateUserAnno}
           handleCancel={this.handleDisableAnnoMode}
           annoInput={
-            <AnnoInputs
+            <LabelInput
-              text={newCategoryText}
+              labelSuggestions={ontologyEnabled ? ontology.terms : null}
-              handleItemChange={this.handleSuggestActiveItemChange}
+              onChange={this.handleChange}
-              handleChoice={this.handleChoice}
+              onSelect={this.handleSelect}
-              handleTextChange={this.handleNewCategoryText}
+              inputProps={{
+                "data-testid": "new-category-name",
+                leftIcon: "tag",
+                intent: "none",
+                autoFocus: true
+              }}
+              newLabelMessage="New category"
             />
           }
           annoSelect={
@@ -173,6 +183,8 @@ class Categories extends React.Component {
             <Category
               key={catName}
               metadataField={catName}
+              onExpansionChange={this.onExpansionChange}
+              isExpanded={expandedCats.has(catName)}
               createAnnoModeActive={createAnnoModeActive}
               isUserAnno={false}
             />
@@ -184,6 +196,8 @@ class Categories extends React.Component {
             <Category
               key={catName}
               metadataField={catName}
+              onExpansionChange={this.onExpansionChange}
+              isExpanded={expandedCats.has(catName)}
               createAnnoModeActive={createAnnoModeActive}
               isUserAnno
             />
@@ -191,7 +205,11 @@ class Categories extends React.Component {
         )}
         {writableCategoriesEnabled ? (
           <div>
-            <Button onClick={this.handleEnableAnnoMode} intent="primary">
+            <Button
+              data-testid="open-annotation-dialog"
+              onClick={this.handleEnableAnnoMode}
+              intent="primary"
+            >
               Create new category
             </Button>
           </div>

client/src/components/categorical/category.js

@@ -2,37 +2,37 @@ import React from "react";
 import _ from "lodash";
 import { connect } from "react-redux";
 import { FaChevronRight, FaChevronDown } from "react-icons/fa";
-import { Button, Tooltip, Icon } from "@blueprintjs/core";
+import { AnchorButton, Button, Tooltip, Icon } from "@blueprintjs/core";
 import CategoryFlipperLayout from "./categoryFlipperLayout";
 import AnnoMenu from "./annoMenuCategory";
 import AnnoDialogEditCategoryName from "./annoDialogEditCategoryName";
 import AnnoDialogAddLabel from "./annoDialogAddLabel";
-import AnnoDialogAddLabelFromOntology from "./annoDialogAddLabelFromOntology";
 
 import * as globals from "../../globals";
+import maybeTruncateString from "../../util/maybeTruncateString";
 
 @connect(state => ({
   colorAccessor: state.colors.colorAccessor,
   categoricalSelection: state.categoricalSelection,
   annotations: state.annotations,
-  universe: state.universe,
+  universe: state.universe
-  ontology: state.ontology,
-  ontologyLoading: state.ontology?.loading,
-  ontologyEnabled: state.ontology?.enabled
 }))
 class Category extends React.Component {
   constructor(props) {
     super(props);
     this.state = {
-      isChecked: true,
+      isChecked: true
-      isExpanded: false
     };
   }
 
   componentDidUpdate(prevProps) {
     const { categoricalSelection, metadataField } = this.props;
-    if (categoricalSelection !== prevProps.categoricalSelection) {
+    const cat = categoricalSelection?.[metadataField];
-      const cat = categoricalSelection[metadataField];
+    if (
+      categoricalSelection !== prevProps.categoricalSelection &&
+      !!cat &&
+      !!this.checkbox
+    ) {
       const categoryCount = {
         // total number of categories in this dimension
         totalCatCount: cat.numCategoryValues,
@@ -90,20 +90,80 @@ class Category extends React.Component {
     // || this.checkbox.indeterminate === false
     if (isChecked) {
       this.toggleNone();
-    } else if (!isChecked) {
+    } else {
       this.toggleAll();
     }
   }
 
+  renderIsStillLoading() {
+    /*
+    We are still loading this category, so render a "busy" signal.
+    */
+    const { metadataField } = this.props;
+    return (
+      <div
+        style={{
+          maxWidth: globals.maxControlsWidth
+        }}
+      >
+        <div
+          style={{
+            display: "flex",
+            justifyContent: "space-between",
+            alignItems: "baseline"
+          }}
+        >
+          <div
+            style={{
+              display: "flex",
+              justifyContent: "flex-start",
+              alignItems: "flex-start"
+            }}
+          >
+            <label className="bp3-control bp3-checkbox">
+              <input disabled checked={true} type="checkbox" />
+              <span className="bp3-control-indicator" />
+            </label>
+            <span
+              style={{
+                cursor: "pointer",
+                display: "inline-block"
+              }}
+            >
+              {metadataField}
+            </span>
+          </div>
+          <div>
+            <Button minimal loading intent="primary" />
+          </div>
+        </div>
+      </div>
+    );
+  }
+
   render() {
-    const { isExpanded, isChecked } = this.state;
+    const { isChecked } = this.state;
     const {
       metadataField,
+      categoricalSelection,
       colorAccessor,
       isUserAnno,
-      annotations
+      annotations,
+      isExpanded,
+      onExpansionChange
     } = this.props;
 
+    const isStillLoading = !(categoricalSelection?.[metadataField] ?? false);
+    if (isStillLoading) {
+      return this.renderIsStillLoading();
+    }
+
+    const isTruncated = _.get(
+      categoricalSelection,
+      [metadataField, "isTruncated"],
+      false
+    );
+
     return (
       <CategoryFlipperLayout
         metadataField={metadataField}
@@ -120,7 +180,7 @@ class Category extends React.Component {
           <label className="bp3-control bp3-checkbox">
             <input
               data-testclass="category-select"
-              data-testid={`category-select-${metadataField}`}
+              data-testid={`${metadataField}:category-select`}
               onChange={this.handleToggleAllClick.bind(this)}
               ref={el => {
                 this.checkbox = el;
@@ -132,7 +192,7 @@ class Category extends React.Component {
             <span className="bp3-control-indicator" />
           </label>
           <span
-            data-testid={`category-expand-${metadataField}`}
+            data-testid={`${metadataField}:category-expand`}
             style={{
               cursor: "pointer",
               display: "inline-block"
@@ -142,14 +202,17 @@ class Category extends React.Component {
                 annotations.isEditingCategoryName &&
                 annotations.categoryBeingEdited === metadataField;
               if (!editingCategory) {
-                this.setState({ isExpanded: !isExpanded });
+                onExpansionChange(metadataField);
               }
             }}
           >
             {isUserAnno ? (
               <Icon style={{ marginRight: 5 }} icon="tag" iconSize={16} />
             ) : null}
-            {metadataField}
+            {maybeTruncateString(
+              metadataField,
+              globals.categoryDisplayStringMaxLength
+            )}
             {isExpanded ? (
               <FaChevronDown
                 data-testclass="category-expand-is-expanded"
@@ -165,28 +228,32 @@ class Category extends React.Component {
         </div>
         {<AnnoDialogEditCategoryName metadataField={metadataField} />}
         {<AnnoDialogAddLabel metadataField={metadataField} />}
-        {<AnnoDialogAddLabelFromOntology metadataField={metadataField} />}
         <div>
           <AnnoMenu
             metadataField={metadataField}
             isUserAnno={isUserAnno}
             createText="Add a new label to this category"
-            createFromOntologyText="Add a new label to this category using existing ontology terms"
             editText="Edit this category's name"
             deleteText="Delete this category, all associated labels, and remove all cell assignments"
           />
 
           <Tooltip
-            content="Use as color scale"
+            content={
+              isTruncated
+                ? `Coloring by ${metadataField} is disabled, as it exceeds the limit of ${globals.maxCategoricalOptionsToDisplay} labels`
+                : "Use as color scale"
+            }
             position="bottom"
+            usePortal={false}
             hoverOpenDelay={globals.tooltipHoverOpenDelay}
           >
-            <Button
+            <AnchorButton
               data-testclass="colorby"
               data-testid={`colorby-${metadataField}`}
               onClick={this.handleColorChange}
               active={colorAccessor === metadataField}
               intent={colorAccessor === metadataField ? "primary" : "none"}
+              disabled={isTruncated}
               icon="tint"
             />
           </Tooltip>

client/src/components/categorical/labelInput.js

@@ -0,0 +1,176 @@
+import React from "react";
+import { InputGroup, MenuItem, Keys } from "@blueprintjs/core";
+import { Suggest } from "@blueprintjs/select";
+import fuzzysort from "fuzzysort";
+
+export default class LabelInput extends React.PureComponent {
+  /*
+  Input widget for text labels, which acts like an InputGroup, but will also 
+  accept a suggestion list (of labels), with sublime-like suggest search.
+
+  Properties:
+
+  * labelSuggestions -- array of suggested lables. May be array of string or
+    other objects. If array of objects, specify `labelKey`. If null, suggestion
+    mode is disabled.
+  * onSelect -- optional, callback upon selection of item from labelSuggestions.
+    (label) => void
+  * onChange -- optional, callback upon change in text input.  (label) => void
+  * label -- component value, for controlled use
+  * newLabelMessage -- text to display when user enters a label not in labelSuggestions
+    (not used if suggestion mode disabled)
+  * inputProps -- will be passed to InputGroup
+  * popoverProps -- will be passed to <Suggest>
+  */
+
+  constructor(props) {
+    super(props);
+
+    const { label } = props;
+    const query = label || "";
+    const queryResults = this.filterLabels(query);
+    this.state = {
+      query,
+      queryResults
+    };
+  }
+
+  handleQueryChange = (query, event) => {
+    // https://github.com/palantir/blueprint/issues/2983
+    if (!event) return;
+
+    const queryResults = this.filterLabels(query);
+    this.setState({
+      query,
+      queryResults
+    });
+
+    const { onChange } = this.props;
+    if (onChange) onChange(query, event);
+  };
+
+  handleItemSelect = (item, event) => {
+    /* only report the select if not already reported via onChange() */
+    const { target } = item;
+    const { query } = this.state;
+    const { onSelect } = this.props;
+    if (target !== query && onSelect) onSelect(target, event);
+  };
+
+  handleKeyDown = e => {
+    /* 
+    prevent these events from propagating to containing form/dialog
+    and causing further side effects (eg, closing dialog, submitting
+    form, etc).
+    */
+    const { keyCode } = e;
+    if (keyCode === Keys.ENTER || keyCode === Keys.ESCAPE) {
+      e.preventDefault();
+    }
+    if (keyCode === Keys.ESCAPE) {
+      e.stopPropagation();
+    }
+  };
+
+  handleChange = e => {
+    const { onChange } = this.props;
+    if (onChange) onChange(e.target.value);
+  };
+
+  renderLabelSuggestion = (queryResult, { handleClick, modifiers }) => {
+    if (queryResult.newLabel) {
+      const { newLabelMessage } = this.props;
+      return (
+        <MenuItem
+          icon="flag"
+          active={modifiers.active}
+          disabled={modifiers.disabled}
+          key={queryResult.target}
+          onClick={handleClick}
+          text={<em>{queryResult.target}</em>}
+          label={newLabelMessage || "New label"}
+        />
+      );
+    }
+    return (
+      <MenuItem
+        active={modifiers.active}
+        disabled={modifiers.disabled}
+        key={queryResult.target}
+        onClick={handleClick}
+        text={queryResult.target}
+      />
+    );
+  };
+
+  /* maxinum number of suggestions */
+  static QueryResultLimit = 100;
+
+  filterLabels(query) {
+    const { labelSuggestions } = this.props;
+    if (!labelSuggestions) return [];
+
+    /* empty query is wildcard */
+    if (query === "") {
+      return labelSuggestions.slice(0, LabelInput.QueryResultLimit).map(l => ({
+        target: l,
+        score: -10000
+      }));
+    }
+
+    /* else, do a fuzzy query */
+    const options = {
+      limit: LabelInput.QueryResultLimit,
+      threshold: -10000 // don't return bad results
+    };
+    let queryResults = fuzzysort.go(query, labelSuggestions, options);
+    /* exact match will always be first in list */
+    if (query !== "" && queryResults[0]?.target !== query)
+      queryResults = [{ target: query, newLabel: true }, ...queryResults];
+
+    return queryResults;
+  }
+
+  render() {
+    const { props } = this;
+    const { labelSuggestions, label, autoFocus = true } = props;
+    const suggestEnabled = !!labelSuggestions && labelSuggestions.length > 0;
+
+    if (!suggestEnabled) {
+      return (
+        <InputGroup
+          autoFocus={autoFocus}
+          {...props.inputProps} // eslint-disable-line react/jsx-props-no-spreading
+          value={label}
+          onChange={this.handleChange}
+        />
+      );
+    }
+
+    const popoverProps = {
+      minimal: true,
+      ...props.popoverProps
+    };
+    const inputProps = {
+      ...props.inputProps,
+      autoFocus: false
+    };
+    const { queryResults } = this.state;
+    return (
+      <>
+        <Suggest
+          fill
+          inputValueRenderer={i => i.target}
+          items={queryResults}
+          itemRenderer={this.renderLabelSuggestion}
+          onItemSelect={this.handleItemSelect}
+          query={label}
+          onQueryChange={this.handleQueryChange}
+          popoverProps={popoverProps}
+          inputProps={inputProps}
+          onKeyDown={this.handleKeyDown}
+        />
+      </>
+    );
+  }
+}

client/src/components/categorical/labelUtil.js

@@ -1,4 +1,6 @@
 import React from "react";
+import { Colors } from "@blueprintjs/core";
+
 import { AnnotationsHelpers } from "../../util/stateManager";
 
 export function isLabelErroneous(label, metadataField, ontology, schema) {
@@ -24,34 +26,38 @@ export function isLabelErroneous(label, metadataField, ontology, schema) {
 	return false;
 }
 
-export function labelErrorMessage(label, metadataField, ontology, schema) {
+/* all other errors - map code to human error message */
-	const err = isLabelErroneous(label, metadataField, ontology, schema);
+const errorMessageMap = {
+	"empty-string": "Blank names not allowed",
+	duplicate: "Name must be unique",
+	"trim-spaces": "Leading and trailing spaces not allowed",
+	"illegal-characters":
+		"Only alphanumeric and special characters (-_.) allowed",
+	"multi-space-run": "Multiple consecutive spaces not allowed"
+};
 
-	if (err === "duplicate") {
+export function labelPrompt(err, prolog, epilog) {
-		/* duplicate error is special cased because it has special formatting */
+	let errPrompt = null;
-		return (
+	if (err) {
-			<span>
+		let errMsg = errorMessageMap[err] ?? "error";
-				<span style={{ fontStyle: "italic" }}>{label}</span> already
+		errMsg = errMsg[0].toLowerCase() + errMsg.slice(1);
-				exists already exists within{" "}
+		errPrompt = (
-				<span style={{ fontStyle: "italic" }}>{metadataField}</span>{" "}
+			<span
+				style={{
+					marginTop: 7,
+					color: Colors.ORANGE3
+				}}
+			>
+				{errMsg}
 			</span>
 		);
 	}
-
+	return (
-	if (err) {
+		<span>
-		/* all other errors - map code to human error message */
+			{prolog}
-		const errorMessageMap = {
+			{err ? " - " : null}
-			"empty-string": "Blank names not allowed",
+			{errPrompt}
-			duplicate: "Name must be unique",
+			{epilog}
-			"trim-spaces": "Leading and trailing spaces not allowed",
+		</span>
-			"illegal-characters":
+	);
-				"Only alphanumeric and special characters (-_.) allowed",
-			"multi-space-run": "Multiple consecutive spaces not allowed"
-		};
-		const errorMessage = errorMessageMap[err] ?? "error";
-		return <span>{errorMessage}</span>;
-	}
-
-	/* no error, no message generated */
-	return null;
 }

client/src/components/categorical/ontologySelect.js

@@ -1,57 +0,0 @@
-import React from "react";
-import { connect } from "react-redux";
-import { Button, MenuItem } from "@blueprintjs/core";
-import { Select } from "@blueprintjs/select";
-import fuzzysort from "fuzzysort";
-
-const filterOntology = (query, ontology) =>
-  /* fires on load, once, and then for each character typed into the input */
-  fuzzysort.go(query, ontology, {
-    limit: 100,
-    threshold: -10000 // don't return bad results
-  });
-
-@connect()
-class ChooseOntologySelect extends React.Component {
-  constructor(props) {
-    super(props);
-    this.state = {};
-  }
-
-  render() {
-    const {
-      ontology,
-      categoryToDuplicate,
-      handleChooseOntologyTermFromDropdown
-    } = this.props;
-    return (
-      <div>
-        <Select
-          items={
-            ontology?.terms ||
-            [] /* this is a placeholder, could be  a subcomponent to avoid this */
-          }
-          filterable
-          itemListPredicate={filterOntology}
-          itemRenderer={(d, { handleClick }) => {
-            return (
-              <MenuItem onClick={handleClick} key={d.target} text={d.target} />
-            );
-          }}
-          noResults={<MenuItem disabled text="No results." />}
-          onItemSelect={d => {
-            handleChooseOntologyTermFromDropdown(d);
-          }}
-        >
-          {/* children become the popover target; render value here */}
-          <Button
-            text={categoryToDuplicate || "Choose an Ontology Term"}
-            rightIcon="double-caret-vertical"
-          />
-        </Select>
-      </div>
-    );
-  }
-}
-
-export default ChooseOntologySelect;

client/src/components/categorical/value.js

@@ -1,4 +1,3 @@
-// jshint esversion: 6
 import { connect } from "react-redux";
 import React from "react";
 
@@ -9,40 +8,49 @@ import {
   Popover,
   Position,
   PopoverInteractionKind,
-  Tooltip,
+  Tooltip
-  Colors
 } from "@blueprintjs/core";
 import Occupancy from "./occupancy";
 import * as globals from "../../globals";
 import styles from "./categorical.css";
 import AnnoDialog from "./annoDialog";
-import AnnoInputs from "./annoInputs";
+import LabelInput from "./labelInput";
 
 import { AnnotationsHelpers } from "../../util/stateManager";
-import { labelErrorMessage, isLabelErroneous } from "./labelUtil";
+import { labelPrompt, isLabelErroneous } from "./labelUtil";
 
-@connect(state => ({
+/* this is defined outside of the class so we can use it in connect() */
-  categoricalSelection: state.categoricalSelection,
+function _currentLabel(ownProps, categoricalSelection) {
-  annotations: state.annotations,
+  const { metadataField, categoryIndex } = ownProps;
-  colorScale: state.colors.scale,
+  return String(
-  colorAccessor: state.colors.colorAccessor,
+    categoricalSelection[metadataField].categoryValues[categoryIndex]
-  pointDilation: state.pointDilation,
+  ).valueOf();
-  schema: state.world?.schema,
+}
-  world: state.world,
+
-  crossfilter: state.crossfilter,
+@connect((state, ownProps) => {
-  ontology: state.ontology,
+  const { pointDilation, categoricalSelection } = state;
-  ontologyLoading: state.ontology?.loading,
+  const { metadataField, categoryField } = ownProps;
-  ontologyEnabled: state.ontology?.enabled
+  const isDilated =
-}))
+    pointDilation.metadataField === metadataField &&
+    pointDilation.categoryField ===
+      _currentLabel(ownProps, categoricalSelection);
+  return {
+    categoricalSelection,
+    annotations: state.annotations,
+    colorScale: state.colors.scale,
+    colorAccessor: state.colors.colorAccessor,
+    schema: state.world?.schema,
+    world: state.world,
+    crossfilter: state.crossfilter,
+    ontology: state.ontology,
+    isDilated
+  };
+})
 class CategoryValue extends React.Component {
   constructor(props) {
     super(props);
     this.state = {
-      editedLabelText: String(
+      editedLabelText: this.currentLabel()
-        props.categoricalSelection[props.metadataField].categoryValues[
-          props.categoryIndex
-        ]
-      ).valueOf()
     };
   }
 
@@ -54,9 +62,7 @@ class CategoryValue extends React.Component {
       prevProps.categoryIndex !== categoryIndex
     ) {
       this.setState({
-        editedLabelText: String(
+        editedLabelText: this.currentLabel()
-          categoricalSelection[metadataField].categoryValues[categoryIndex]
-        ).valueOf()
       });
     }
   }
@@ -83,10 +89,11 @@ class CategoryValue extends React.Component {
     });
   };
 
-  handleEditValue = () => {
+  handleEditValue = e => {
     const { dispatch, metadataField, categoryIndex } = this.props;
     const { editedLabelText } = this.state;
     const label = this.getLabel();
+    this.cancelEditMode();
     dispatch({
       type: "annotation: label edited",
       editedLabel: editedLabelText,
@@ -94,51 +101,30 @@ class CategoryValue extends React.Component {
       categoryIndex,
       label
     });
+    e.preventDefault();
   };
 
-  handleCreateArbitraryLabel = editedLabelTextNotInOntology => {
+  handleCreateArbitraryLabel = txt => {
     const { dispatch, metadataField, categoryIndex } = this.props;
     const label = this.getLabel();
-
+    this.cancelEditMode();
     dispatch({
       type: "annotation: label edited",
       metadataField,
-      editedLabel: editedLabelTextNotInOntology,
+      editedLabel: txt,
       categoryIndex,
       label
     });
   };
 
   labelNameError = name => {
-    const {
+    const { metadataField, ontology, schema } = this.props;
-      metadataField,
+    if (name === this.currentLabel()) return false;
-      ontology,
-      schema,
-      categoricalSelection,
-      categoryIndex
-    } = this.props;
-    const category = categoricalSelection[metadataField];
-    const displayString = String(
-      category.categoryValues[categoryIndex]
-    ).valueOf();
-    if (name === displayString) return false;
     return isLabelErroneous(name, metadataField, ontology, schema);
   };
 
-  labelNameErrorMessage = name => {
+  instruction = label => {
-    const {
+    return labelPrompt(this.labelNameError(label), "New, unique label", ":");
-      metadataField,
-      ontology,
-      schema,
-      categoricalSelection,
-      categoryIndex
-    } = this.props;
-    const category = categoricalSelection[metadataField];
-    const displayString = String(
-      category.categoryValues[categoryIndex]
-    ).valueOf();
-    if (name === displayString) return null;
-    return labelErrorMessage(name, metadataField, ontology, schema);
   };
 
   activateEditLabelMode = () => {
@@ -150,8 +136,11 @@ class CategoryValue extends React.Component {
     });
   };
 
-  cancelEdit = () => {
+  cancelEditMode = () => {
     const { dispatch, metadataField, categoryIndex } = this.props;
+    this.setState({
+      editedLabelText: this.currentLabel()
+    });
     dispatch({
       type: "annotation: cancel edit label mode",
       metadataField,
@@ -196,7 +185,7 @@ class CategoryValue extends React.Component {
     const crossfilterChange =
       props.isUserAnno && props.crossfilter !== nextProps.crossfilter;
     const editingLabel = state.editedLabelText !== nextState.editedLabelText;
-    const dilationChange = props.pointDilation !== nextProps.pointDilation;
+    const dilationChange = props.isDilated !== nextProps.isDilated;
 
     return (
       valueSelectionChange ||
@@ -253,6 +242,11 @@ class CategoryValue extends React.Component {
     return label;
   };
 
+  currentLabel() {
+    const { categoricalSelection } = this.props;
+    return _currentLabel(this.props, categoricalSelection);
+  }
+
   isAddCurrentSelectionDisabled(category, value) {
     /*
     disable "add current selection to label", if one of the following is true:
@@ -292,12 +286,13 @@ class CategoryValue extends React.Component {
       schema,
       isUserAnno,
       annotations,
-      ontologyEnabled,
+      ontology,
       // flippedProps is potentially brittle, their docs want {...flippedProps} on our div,
       // our lint doesn't like jsx spread, we are version pinned to prevent api change on their part
       flippedProps,
-      pointDilation
+      isDilated
     } = this.props;
+    const ontologyEnabled = ontology?.enabled ?? false;
 
     const { editedLabelText } = this.state;
 
@@ -307,9 +302,7 @@ class CategoryValue extends React.Component {
     const selected = category.categoryValueSelected[categoryIndex];
     const count = category.categoryValueCounts[categoryIndex];
     const value = category.categoryValues[categoryIndex];
-    const displayString = String(
+    const displayString = this.currentLabel();
-      category.categoryValues[categoryIndex]
-    ).valueOf();
 
     /* this is the color scale, so add swatches below */
     const isColorBy = metadataField === colorAccessor;
@@ -357,12 +350,7 @@ class CategoryValue extends React.Component {
         data-portal-key={flippedProps["data-portal-key"]}
         className={
           /* This code is to change the styles on centroid label hover is causing over-rendering */
-          `${styles.value}${
+          `${styles.value}${isDilated ? ` ${styles.hover}` : ""}`
-            pointDilation.metadataField === metadataField &&
-            pointDilation.categoryField === displayString
-              ? ` ${styles.hover}`
-              : ""
-          }`
         }
         data-testclass="categorical-row"
         style={{
@@ -433,28 +421,33 @@ class CategoryValue extends React.Component {
               <div>
                 <AnnoDialog
                   isActive={editModeActive}
+                  inputProps={{
+                    "data-testid": `${metadataField}:edit-label-name-dialog`
+                  }}
+                  primaryButtonProps={{
+                    "data-testid": `${metadataField}:${displayString}:submit-label-edit`
+                  }}
                   title="Edit label"
-                  instruction={`New label text must be unique within category ${metadataField}:`}
+                  instruction={this.instruction(editedLabelText)}
                   cancelTooltipContent="Close this dialog without editing label text."
                   primaryButtonText="Change label text"
                   text={editedLabelText}
                   categoryToDuplicate={null}
                   validationError={this.labelNameError(editedLabelText)}
-                  errorMessage={this.labelNameErrorMessage(editedLabelText)}
                   handleSubmit={this.handleEditValue}
-                  handleCancel={this.cancelEdit}
+                  handleCancel={this.cancelEditMode}
                   annoInput={
-                    <AnnoInputs
+                    <LabelInput
-                      useSuggest={ontologyEnabled}
+                      label={editedLabelText}
-                      text={editedLabelText}
+                      labelSuggestions={ontologyEnabled ? ontology.terms : null}
-                      handleCreateArbitraryLabel={
+                      onChange={this.handleTextChange}
-                        this.handleCreateArbitraryLabel
+                      onSelect={this.handleTextChange}
-                      }
+                      inputProps={{
-                      handleItemChange={this.handleSuggestActiveItemChange}
+                        "data-testid": `${metadataField}:${displayString}:edit-label-name`,
-                      handleChoice={this.handleChoice}
+                        leftIcon: "tag",
-                      handleTextChange={this.handleTextChange}
+                        intent: "none",
-                      isTextInvalid={this.labelNameError}
+                        autoFocus: true
-                      isTextInvalidErrorMessage={this.labelNameErrorMessage}
+                      }}
                     />
                   }
                   annoSelect={null}
@@ -464,7 +457,10 @@ class CategoryValue extends React.Component {
           </div>
           <span style={{ flexShrink: 0 }}>
             {colorAccessor && !isColorBy && !annotations.isEditingLabelName ? (
-              <Occupancy category={category} {...this.props} />
+              <Occupancy
+                category={category}
+                {...this.props} // eslint-disable-line react/jsx-props-no-spreading
+              />
             ) : null}
           </span>
         </div>
@@ -513,7 +509,7 @@ class CategoryValue extends React.Component {
                       <MenuItem
                         icon="plus"
                         data-testclass="handleAddCurrentSelectionToThisLabel"
-                        data-testid={`handleAddCurrentSelectionToThisLabel-${metadataField}`}
+                        data-testid={`${metadataField}:${displayString}:add-current-selection-to-this-label`}
                         onClick={this.handleAddCurrentSelectionToThisLabel}
                         text={
                           <span>
@@ -541,7 +537,7 @@ class CategoryValue extends React.Component {
                           icon="edit"
                           text="Edit this label's name"
                           data-testclass="handleEditValue"
-                          data-testid={`handleEditValue-${metadataField}`}
+                          data-testid={`${metadataField}:${displayString}:edit-label`}
                           onClick={this.activateEditLabelMode}
                           disabled={annotations.isEditingLabelName}
                         />
@@ -551,7 +547,7 @@ class CategoryValue extends React.Component {
                           icon="delete"
                           intent="danger"
                           data-testclass="handleDeleteValue"
-                          data-testid={`handleDeleteValue-${metadataField}`}
+                          data-testid={`${metadataField}:${displayString}:delete-label`}
                           onClick={this.handleDeleteValue}
                           text={`Delete this label, and reassign all cells to type '${globals.unassignedCategoryLabel}'`}
                         />
@@ -567,7 +563,7 @@ class CategoryValue extends React.Component {
                       minHeight: 16
                     }}
                     data-testclass="seeActions"
-                    data-testid={`seeActions-${metadataField}`}
+                    data-testid={`${metadataField}:${displayString}:see-actions`}
                     icon="more"
                     small
                     minimal

client/src/components/continuous/continuous.js

@@ -2,10 +2,10 @@
 /* rc slider https://www.npmjs.com/package/rc-slider */
 
 import React from "react";
-import _ from "lodash";
 import { connect } from "react-redux";
-import * as globals from "../../globals";
+import { Button } from "@blueprintjs/core";
 
+import * as globals from "../../globals";
 import HistogramBrush from "../brushableHistogram";
 
 @connect(state => ({
@@ -14,17 +14,7 @@ import HistogramBrush from "../brushableHistogram";
   colorScale: state.colors.scale,
   schema: state.world?.schema
 }))
-class Continuous extends React.Component {
+class Continuous extends React.PureComponent {
-  constructor(props) {
-    super(props);
-    this.hasContinuous = false;
-    this.continuousChecked = false;
-
-    this.state = {};
-  }
-
-  componentDidUpdate() {}
-
   handleColorAction = key => {
     return () => {
       const { dispatch, obsAnnotations } = this.props;
@@ -37,61 +27,84 @@ class Continuous extends React.Component {
     };
   };
 
+  static renderIsStillLoading(zebra, key) {
+    return (
+      <div
+        key={key}
+        style={{
+          padding: globals.leftSidebarSectionPadding,
+          backgroundColor: zebra % 2 === 0 ? globals.lightestGrey : "white"
+        }}
+      >
+        <div
+          style={{
+            display: "flex",
+            justifyContent: "space-between",
+            justifyItems: "center",
+            alignItems: "center"
+          }}
+        >
+          <div style={{ minWidth: 30 }} />
+          <div style={{ display: "flex", alignSelf: "center" }}>
+            <span style={{ fontStyle: "italic" }}>{key}</span>
+          </div>
+          <div
+            style={{
+              display: "flex",
+              justifyContent: "flex-end"
+            }}
+          >
+            <Button minimal loading intent="primary" />
+          </div>
+        </div>
+      </div>
+    );
+  }
+
   render() {
     const { obsAnnotations, schema } = this.props;
-    if (schema && !this.continuousChecked) {
+
-      this.hasContinuous = _.some(
+    const obsIndex = schema.annotations.obs.index;
-        schema.annotations.obs,
+    const allContinuousNames = schema.annotations.obs.columns
-        d => d.type === "int32" || d.type === "float32"
+      .filter(col => col.type === "int32" || col.type === "float32")
-      );
+      .filter(col => col.name !== obsIndex)
-      this.continuousChecked = true; /* only do this once */
+      .map(col => col.name);
-    }
 
     /* initial value for iterator to simulate index, ranges is an object */
     let zebra = 0;
 
     return (
       <div>
-        {this.hasContinuous ? (
+        {allContinuousNames.map(key => {
-          <p
+          if (!obsAnnotations.hasCol(key)) {
-            style={{
+            // still loading!
-              ...globals.leftSidebarSectionHeading,
+            zebra += 1;
-              marginTop: 40,
+            return Continuous.renderIsStillLoading(zebra, key);
-              paddingLeft: globals.leftSidebarSectionPadding
+          }
-            }}
-          >
-            Continuous metadata
-          </p>
-        ) : null}
-        {obsAnnotations
-          ? _.map(obsAnnotations.colIndex.keys(), key => {
-              const isColorField =
-                key.includes("color") || key.includes("Color");
-              if (key === schema.annotations.obs.index || isColorField)
-                return null;
 
-              const summary = obsAnnotations.col(key).summarize();
+          // data loaded and available
-              const nonFiniteExtent =
+          const summary = obsAnnotations.col(key).summarize();
-                summary.min === undefined ||
+          const nonFiniteExtent =
-                summary.max === undefined ||
+            summary.min === undefined ||
-                Number.isNaN(summary.min) ||
+            summary.max === undefined ||
-                Number.isNaN(summary.max);
+            Number.isNaN(summary.min) ||
-              if (!summary.categorical && !nonFiniteExtent) {
+            Number.isNaN(summary.max);
-                zebra += 1;
+          if (!summary.categorical && !nonFiniteExtent) {
-                return (
+            zebra += 1;
-                  <HistogramBrush
+            return (
-                    key={key}
+              <HistogramBrush
-                    field={key}
+                key={key}
-                    isObs
+                field={key}
-                    zebra={zebra % 2 === 0}
+                isObs
-                    ranges={summary}
+                zebra={zebra % 2 === 0}
-                    handleColorAction={this.handleColorAction(key)}
+                ranges={summary}
-                  />
+                handleColorAction={this.handleColorAction(key)}
-                );
+              />
-              }
+            );
-              return null;
+          }
-            })
+
-          : null}
+          return null;
+        })}
       </div>
     );
   }

client/src/components/framework/toasters.js

@@ -2,7 +2,7 @@ import { Position, Toaster, Intent } from "@blueprintjs/core";
 
 /** Singleton toaster instance. Create separate instances for different options. */
 
-const ErrorToastTopCenter = Toaster.create({
+const ToastTopCenter = Toaster.create({
   className: "recipe-toaster",
   position: Position.TOP
 });
@@ -11,21 +11,38 @@ const ErrorToastTopCenter = Toaster.create({
 A "user" error - eg, bad input
 */
 export const postUserErrorToast = message =>
-  ErrorToastTopCenter.show({ message, intent: Intent.WARNING });
+  ToastTopCenter.show({ message, intent: Intent.WARNING });
 
 /*
 A toast the user must dismiss manually, because they need to act on its information,
 ie., 8 bulk add genes out of 40 were bad. Manually see which ones and fix.
 */
 export const keepAroundErrorToast = message =>
-  ErrorToastTopCenter.show({ message, timeout: 0, intent: Intent.WARNING });
+  ToastTopCenter.show({ message, timeout: 0, intent: Intent.WARNING });
 
 /*
 a hard network error
 */
 export const postNetworkErrorToast = message =>
-  ErrorToastTopCenter.show({
+  ToastTopCenter.show({
     message,
     timeout: 30000,
     intent: Intent.DANGER
   });
+
+/*
+Async message to user
+*/
+export const postAsyncSuccessToast = message =>
+  ToastTopCenter.show({
+    message,
+    timeout: 10000,
+    intent: Intent.SUCCESS
+  });
+
+export const postAsyncFailureToast = message =>
+  ToastTopCenter.show({
+    message,
+    timeout: 10000,
+    intent: Intent.WARNING
+  });

client/src/components/geneExpression/index.js

@@ -103,7 +103,8 @@ class GeneExpression extends React.Component {
       if (genes.length === 0) {
         return keepAroundErrorToast("Must enter a gene name.");
       }
-      const worldGenes = world.varAnnotations.col(varIndexName).asArray();
+      const worldGenes =
+        world.varAnnotations?.col(varIndexName)?.asArray() || [];
 
       // These gene lists are unique enough where memoization is useless
       const upperGenes = this._genesToUpper(genes);
@@ -181,9 +182,9 @@ class GeneExpression extends React.Component {
     const gene = g.target;
     if (userDefinedGenes.indexOf(gene) !== -1) {
       postUserErrorToast("That gene already exists");
-    } else if (userDefinedGenes.length > 15) {
+    } else if (userDefinedGenes.length > globals.maxUserDefinedGenes) {
       postUserErrorToast(
-        "That's too many genes, you can have at most 15 user defined genes"
+        `That's too many genes, you can have at most ${globals.maxUserDefinedGenes} user defined genes`
       );
     } else if (
       world.varAnnotations.col(varIndexName).indexOf(gene) === undefined
@@ -206,8 +207,12 @@ class GeneExpression extends React.Component {
       differential
     } = this.props;
     const varIndexName = world?.schema?.annotations?.var?.index;
+    const varIndex = world?.varAnnotations?.col(varIndexName)?.asArray();
     const { tab, bulkAdd, activeItem } = this.state;
 
+    // may still be loading!
+    if (!varIndex) return null;
+
     return (
       <div>
         <div>
@@ -268,11 +273,7 @@ class GeneExpression extends React.Component {
                 itemListPredicate={filterGenes}
                 onActiveItemChange={item => this.setState({ activeItem: item })}
                 itemRenderer={renderGene.bind(this)}
-                items={
+                items={varIndex || ["No genes"]}
-                  world && world.varAnnotations
-                    ? world.varAnnotations.col(varIndexName).asArray()
-                    : ["No genes"]
-                }
                 popoverProps={{ minimal: true }}
               />
               <Button

client/src/components/graph/graph.js

@@ -255,7 +255,7 @@ class Graph extends React.Component {
     const { regl, toolSVG, camera, modelTF } = this.state;
     let stateChanges = {};
 
-    if (regl && world) {
+    if (regl && world && crossfilter) {
       /* update the regl and point rendering state */
       const { obsLayout, nObs } = world;
       const { drawPoints, pointBuffer, colorBuffer, flagBuffer } = this.state;
@@ -398,7 +398,7 @@ class Graph extends React.Component {
       // don't return "change" of state unless we are really changing it!
       const { toolSVG } = this.state;
       if (toolSVG === undefined) return {};
-      else return { toolSVG: undefined };
+      return { toolSVG: undefined };
     }
 
     let handleStart;
@@ -422,8 +422,7 @@ class Graph extends React.Component {
       handleEnd,
       handleCancel,
       responsive,
-      this.graphPaddingRightLeft,
+      this.graphPaddingRightLeft
-      graphInteractionMode
     );
 
     return { toolSVG: newToolSVG, tool, container };
@@ -749,6 +748,11 @@ class Graph extends React.Component {
               graphPaddingRightLeft={this.graphPaddingRightLeft}
               graphPaddingTop={this.graphPaddingTop}
               responsive={responsive}
+              handleCanvasEvent={
+                graphInteractionMode === "zoom"
+                  ? this.handleCanvasEvent
+                  : undefined
+              }
             >
               <CentroidLabels />
             </GraphOverlayLayer>

client/src/components/graph/overlays/centroidLabels.js

@@ -1,7 +1,10 @@
+/* eslint-disable max-classes-per-file */
 /* eslint-disable jsx-a11y/mouse-events-have-key-events */
 import React, { PureComponent } from "react";
 import { connect } from "react-redux";
 
+import { categoryLabelDisplayStringLongLength } from "../../../globals";
+
 export default
 @connect(state => ({
   colorAccessor: state.colors.colorAccessor,
@@ -9,6 +12,21 @@ export default
   labels: state.centroidLabels.labels
 }))
 class CentroidLabels extends PureComponent {
+  // Check to see if centroids have either just been displayed or removed from the overlay
+  componentDidUpdate = prevProps => {
+    const { labels, overlayToggled } = this.props;
+    const prevSize = prevProps.labels.size;
+    const { size } = labels;
+
+    const displayChangeOff = prevSize > 0 && size === undefined;
+    const displayChangeOn = prevSize === undefined && size > 0;
+
+    if (displayChangeOn || displayChangeOff) {
+      // Notify overlay layer of display change
+      overlayToggled("centroidLabels", displayChangeOn);
+    }
+  };
+
   render() {
     const {
       labels,
@@ -28,12 +46,24 @@ class CentroidLabels extends PureComponent {
         fontSize = "18px";
         fontWeight = "800";
       }
+
+      // Mirror LSB middle truncation
+      let label = key;
+      if (label.length > categoryLabelDisplayStringLongLength) {
+        label = `${key.slice(
+          0,
+          categoryLabelDisplayStringLongLength / 2
+        )}…${key.slice(-categoryLabelDisplayStringLongLength / 2)}`;
+      }
+
       labelSVGS.push(
         <g
           // eslint-disable-next-line react/no-array-index-key
           key={key}
           className="centroid-label"
           transform={`translate(${value[0]}, ${value[1]})`}
+          data-testclass="centroid-label"
+          data-testid={`${key}-centroid-label`}
         >
           <text
             transform={inverseTransform}
@@ -62,7 +92,7 @@ class CentroidLabels extends PureComponent {
             }
             pointerEvents="visiblePainted"
           >
-            {key.length > 20 ? `${key.substr(0, 20)}...` : key}
+            {label}
           </text>
         </g>
       );

client/src/components/graph/overlays/graphOverlayLayer.js

@@ -12,8 +12,16 @@ class GraphOverlayLayer extends PureComponent {
     This component takes its children (assumed in the data coordinate space ([0, 1] range, origin in bottom left corner))
     and transforms itself multiple times resulting in screen space ([0, screenWidth/Height] range, origin in top left corner)
 
-    Children are assigned in the graph component
+    Children are assigned in the graph component and must implement onDisplayChange()
    */
+  constructor(props) {
+    super(props);
+
+    this.state = {
+      display: {}
+    };
+  }
+
   matrixToTransformString = m => {
     /* 
       Translates the gl-matrix mat3 to SVG matrix transform style
@@ -30,6 +38,13 @@ class GraphOverlayLayer extends PureComponent {
     return `matrix(${1 / m[0]} 0 0 ${1 / m[4]} 0 0)`;
   };
 
+  // This is passed to all children, should be called when an overlay's display state is toggled along with the overlay name and its new display state in boolean form
+  overlayToggled = (overlay, displaying) => {
+    this.setState(state => {
+      return { ...state, display: { ...state.display, [overlay]: displaying } };
+    });
+  };
+
   render() {
     const {
       cameraTF,
@@ -38,11 +53,15 @@ class GraphOverlayLayer extends PureComponent {
       responsive,
       graphPaddingRightLeft,
       graphPaddingTop,
-      children
+      children,
+      handleCanvasEvent
     } = this.props;
 
     if (!cameraTF) return null;
 
+    const { display } = this.state;
+    const displaying = Object.values(display).some(value => value); // check to see if at least one overlay is currently displayed
+
     const inverseTransform = `${this.reverseMatrixScaleTransformString(
       modelTF
     )} ${this.reverseMatrixScaleTransformString(
@@ -53,7 +72,13 @@ class GraphOverlayLayer extends PureComponent {
       -(responsive.height - graphPaddingTop)}) scale(2 1) scale(${1 /
       (responsive.width - graphPaddingRightLeft)} 1)`;
 
-    const newChildren = React.Children.toArray(children);
+    // Copy the children passed with the overlay and add the inverse transform and onDisplayChange props
+    const newChildren = React.Children.map(children, child =>
+      cloneElement(child, {
+        inverseTransform,
+        overlayToggled: this.overlayToggled
+      })
+    );
 
     return (
       <svg
@@ -61,7 +86,12 @@ class GraphOverlayLayer extends PureComponent {
         width={responsive.width - graphPaddingRightLeft}
         height={responsive.height}
         pointerEvents="none"
-        style={{ zIndex: 99 }}
+        style={{
+          zIndex: 99,
+          backgroundColor: displaying ? "rgba(255, 255, 255, 0.55)" : ""
+        }}
+        onMouseMove={handleCanvasEvent}
+        onWheel={handleCanvasEvent}
       >
         <g
           id="canvas-transformation-group-x"
@@ -86,9 +116,7 @@ class GraphOverlayLayer extends PureComponent {
                   id="model-transformation-group"
                   transform={this.matrixToTransformString(modelTF)}
                 >
-                  {newChildren.map(child =>
+                  {newChildren}
-                    cloneElement(child, { inverseTransform })
-                  )}
                 </g>
               </g>
             </g>

client/src/components/menubar/clip.js

@@ -116,7 +116,7 @@ function Clip(props) {
               <Button
                 type="button"
                 data-testid="clip-commit"
-                className="bp3-button"
+                intent="primary"
                 disabled={isClipDisabled()}
                 style={{
                   cursor: "pointer",

client/src/components/menubar/embedding.js

@@ -0,0 +1,122 @@
+import React from "react";
+import {
+  AnchorButton,
+  ButtonGroup,
+  Popover,
+  Button,
+  Radio,
+  RadioGroup,
+  Tooltip,
+  Position
+} from "@blueprintjs/core";
+import { connect } from "react-redux";
+import * as globals from "../../globals";
+import { World } from "../../util/stateManager";
+import actions from "../../actions";
+
+@connect(state => ({
+  universe: state.universe,
+  world: state.world,
+  layoutChoice: state.layoutChoice,
+  reembedController: state.reembedController,
+  enableReembedding: state.config?.parameters?.["enable-reembedding"] ?? false
+}))
+class Embedding extends React.PureComponent {
+  handleLayoutChoiceChange = e => {
+    const { dispatch } = this.props;
+    dispatch({
+      type: "set layout choice",
+      layoutChoice: e.currentTarget.value
+    });
+  };
+
+  renderReembedding() {
+    const {
+      enableReembedding,
+      world,
+      universe,
+      dispatch,
+      reembedController
+    } = this.props;
+
+    if (!enableReembedding) return null;
+
+    const loading = !!reembedController?.pendingFetch;
+    const disabled = World.worldEqUniverse(world, universe);
+    const tipContent = disabled
+      ? "Subset cells first, then click to recompute UMAP embedding."
+      : "Click to recompute UMAP embedding on the current cell subset.";
+
+    return (
+      <Tooltip
+        content={tipContent}
+        position="bottom"
+        hoverOpenDelay={globals.tooltipHoverOpenDelay}
+      >
+        <AnchorButton
+          icon="new-object"
+          style={{ marginRight: 10 }}
+          disabled={disabled}
+          onClick={() => dispatch(actions.requestReembed())}
+          loading={loading}
+        />
+      </Tooltip>
+    );
+  }
+
+  render() {
+    const { layoutChoice } = this.props;
+
+    return (
+      <ButtonGroup
+        style={{
+          marginRight: 10
+        }}
+      >
+        <Popover
+          target={
+            <Tooltip
+              content="Select embedding for visualization"
+              position="bottom"
+              hoverOpenDelay={globals.tooltipHoverOpenDelay}
+            >
+              <Button
+                type="button"
+                data-testid="layout-choice"
+                icon="heatmap"
+                style={{
+                  cursor: "pointer"
+                }}
+              />
+            </Tooltip>
+          }
+          position={Position.BOTTOM_RIGHT}
+          content={
+            <div
+              style={{
+                display: "flex",
+                justifyContent: "flex-start",
+                alignItems: "flex-start",
+                flexDirection: "column",
+                padding: 10
+              }}
+            >
+              <RadioGroup
+                label="Embedding Choice"
+                onChange={this.handleLayoutChoiceChange}
+                selectedValue={layoutChoice.current}
+              >
+                {layoutChoice.available.map(name => (
+                  <Radio label={name} value={name} key={name} />
+                ))}
+              </RadioGroup>
+            </div>
+          }
+        />
+        {this.renderReembedding()}
+      </ButtonGroup>
+    );
+  }
+}
+
+export default Embedding;

client/src/components/menubar/index.js

@@ -1,33 +1,21 @@
 // jshint esversion: 6
 import React from "react";
 import { connect } from "react-redux";
-import {
+import { Button, ButtonGroup, AnchorButton, Tooltip } from "@blueprintjs/core";
-  Button,
+import * as globals from "../../globals";
-  ButtonGroup,
-  AnchorButton,
-  Tooltip,
-  Popover,
-  Position,
-  RadioGroup,
-  Radio,
-  Icon
-} from "@blueprintjs/core";
-import { World } from "../../util/stateManager";
 import actions from "../../actions";
 import CellSetButton from "./cellSetButtons";
-import InformationMenu from "./infoMenu";
-import UndoRedoReset from "./undoRedoReset";
 import Clip from "./clip";
-import * as globals from "../../globals";
+import Embedding from "./embedding";
+import InformationMenu from "./infoMenu";
+import Subset from "./subset";
+import UndoRedoReset from "./undoRedo";
 
 @connect(state => ({
   universe: state.universe,
   world: state.world,
-  loading: state.controls.loading,
   crossfilter: state.crossfilter,
   differential: state.differential,
-  resettingInterface: state.controls.resettingInterface,
-  layoutChoice: state.layoutChoice,
   graphInteractionMode: state.controls.graphInteractionMode,
   clipPercentileMin: Math.round(100 * (state.world?.clipQuantiles?.min ?? 0)),
   clipPercentileMax: Math.round(100 * (state.world?.clipQuantiles?.max ?? 1)),
@@ -100,61 +88,6 @@ class MenuBar extends React.Component {
     return isDisabled;
   };
 
-  isResetDisabled = () => {
-    /*
-    Reset should be disabled when all of the following are true:
-      * nothing is selected in the crossfilter
-      * world EQ universe
-      * nothing is colored by
-      * there are no userDefinedGenes or diffexpGenes displayed
-      * scatterplot is not displayed
-      * nothing in cellset1 or cellset2
-      * clip percentiles are [0,100]
-    */
-    const {
-      crossfilter,
-      world,
-      universe,
-      userDefinedGenes,
-      diffexpGenes,
-      colorAccessor,
-      scatterplotXXaccessor,
-      scatterplotYYaccessor,
-      celllist1,
-      celllist2,
-      clipPercentileMin,
-      clipPercentileMax
-    } = this.props;
-
-    if (!crossfilter || !world || !universe) {
-      return false;
-    }
-    const nothingSelected = crossfilter.countSelected() === crossfilter.size();
-    const nothingColoredBy = !colorAccessor;
-    const noGenes = userDefinedGenes.length === 0 && diffexpGenes.length === 0;
-    const scatterNotDpl = !scatterplotXXaccessor || !scatterplotYYaccessor;
-    const nothingInCellsets = !celllist1 && !celllist2;
-
-    return (
-      nothingSelected &&
-      World.worldEqUniverse(world, universe) &&
-      nothingColoredBy &&
-      noGenes &&
-      scatterNotDpl &&
-      nothingInCellsets &&
-      clipPercentileMax === 100 &&
-      clipPercentileMin === 0
-    );
-  };
-
-  resetInterface = () => {
-    const { dispatch } = this.props;
-    dispatch({
-      type: "interface reset started"
-    });
-    dispatch(actions.resetInterface());
-  };
-
   handleClipOnKeyPress = e => {
     /*
     allow only numbers, plus other critical keys which
@@ -229,14 +162,6 @@ class MenuBar extends React.Component {
     this.setState({ pendingClipPercentiles: null });
   };
 
-  handleLayoutChoiceChange = e => {
-    const { dispatch } = this.props;
-    dispatch({
-      type: "set layout choice",
-      layoutChoice: e.currentTarget.value
-    });
-  };
-
   computeDiffExp = () => {
     const { dispatch, differential } = this.props;
     if (differential.celllist1 && differential.celllist2) {
@@ -269,6 +194,19 @@ class MenuBar extends React.Component {
     });
   };
 
+  subsetPossible = () => {
+    const { crossfilter } = this.props;
+    return (
+      crossfilter.countSelected() !== 0 &&
+      crossfilter.countSelected() !== crossfilter.size()
+    );
+  };
+
+  subsetResetPossible = () => {
+    const { world, universe } = this.props;
+    return world.nObs !== universe.nObs;
+  };
+
   renderDiffExp() {
     /* diffexp-related buttons may be disabled */
     const { disableDiffexp, differential, diffexpMayBeSlow } = this.props;
@@ -284,8 +222,14 @@ class MenuBar extends React.Component {
 
     return (
       <ButtonGroup style={{ marginRight: 10 }}>
-        <CellSetButton {...this.props} eitherCellSetOneOrTwo={1} />
+        <CellSetButton
-        <CellSetButton {...this.props} eitherCellSetOneOrTwo={2} />
+          {...this.props} // eslint-disable-line react/jsx-props-no-spreading
+          eitherCellSetOneOrTwo={1}
+        />
+        <CellSetButton
+          {...this.props} // eslint-disable-line react/jsx-props-no-spreading
+          eitherCellSetOneOrTwo={2}
+        />
         {!differential.diffExp ? (
           <Tooltip
             content={tipMessage}
@@ -327,15 +271,12 @@ class MenuBar extends React.Component {
   render() {
     const {
       dispatch,
-      crossfilter,
-      resettingInterface,
       libraryVersions,
       undoDisabled,
       redoDisabled,
       selectionTool,
       clipPercentileMin,
       clipPercentileMax,
-      layoutChoice,
       graphInteractionMode,
       aboutLink,
       showCentroidLabels
@@ -363,29 +304,18 @@ class MenuBar extends React.Component {
         }}
       >
         {this.renderDiffExp()}
-        <Tooltip
+        <Subset
-          content="Show only metadata and cells which are currently selected"
+          subsetPossible={this.subsetPossible()}
-          position="bottom"
+          subsetResetPossible={this.subsetResetPossible()}
-          hoverOpenDelay={globals.tooltipHoverOpenDelay}
+          handleSubset={() => {
-        >
+            dispatch(actions.setWorldToSelection());
-          <AnchorButton
+            dispatch({ type: "increment graph render counter" });
-            data-testid="subset-button"
+          }}
-            disabled={
+          handleSubsetReset={() => {
-              crossfilter &&
+            dispatch(actions.resetWorldToUniverse());
-              (crossfilter.countSelected() === 0 ||
+            dispatch({ type: "increment graph render counter" });
-                crossfilter.countSelected() === crossfilter.size())
+          }}
-            }
+        />
-            style={{
-              marginRight: 10
-            }}
-            onClick={() => {
-              dispatch(actions.regraph());
-              dispatch({ type: "increment graph render counter" });
-            }}
-          >
-            <Icon icon="double-chevron-down" />
-          </AnchorButton>
-        </Tooltip>
         <ButtonGroup style={{ marginRight: "10px" }}>
           <Tooltip
             content={selectionTooltip}
@@ -436,6 +366,8 @@ class MenuBar extends React.Component {
           disabled={graphInteractionMode === "zoom"}
         >
           <Button
+            type="button"
+            data-testid="centroid-label-toggle"
             icon="property"
             onClick={this.handleCentroidChange}
             active={showCentroidLabels}
@@ -445,52 +377,7 @@ class MenuBar extends React.Component {
             }}
           />
         </Tooltip>
-        <ButtonGroup
+        <Embedding />
-          style={{
-            marginRight: 10
-          }}
-        >
-          <Popover
-            target={
-              <Tooltip
-                content="Select embedding for visualization"
-                position="bottom"
-                hoverOpenDelay={globals.tooltipHoverOpenDelay}
-              >
-                <Button
-                  type="button"
-                  data-testid="layout-choice"
-                  icon="heatmap"
-                  style={{
-                    cursor: "pointer"
-                  }}
-                />
-              </Tooltip>
-            }
-            position={Position.BOTTOM_RIGHT}
-            content={
-              <div
-                style={{
-                  display: "flex",
-                  justifyContent: "flex-start",
-                  alignItems: "flex-start",
-                  flexDirection: "column",
-                  padding: 10
-                }}
-              >
-                <RadioGroup
-                  label="Embedding Choice"
-                  onChange={this.handleLayoutChoiceChange}
-                  selectedValue={layoutChoice.current}
-                >
-                  {layoutChoice.available.map(name => (
-                    <Radio label={name} value={name} key={name} />
-                  ))}
-                </RadioGroup>
-              </div>
-            }
-          />
-        </ButtonGroup>
         <Clip
           pendingClipPercentiles={pendingClipPercentiles}
           clipPercentileMin={clipPercentileMin}
@@ -509,9 +396,6 @@ class MenuBar extends React.Component {
         />
         <UndoRedoReset
           dispatch={dispatch}
-          isResetDisabled={this.isResetDisabled}
-          resetInterface={this.resetInterface}
-          resettingInterface={resettingInterface}
           undoDisabled={undoDisabled}
           redoDisabled={redoDisabled}
         />

client/src/components/menubar/subset.js

@@ -0,0 +1,43 @@
+import React from "react";
+import { AnchorButton, ButtonGroup, Tooltip } from "@blueprintjs/core";
+import * as globals from "../../globals";
+
+function Subset(props) {
+  const {
+    subsetPossible,
+    subsetResetPossible,
+    handleSubset,
+    handleSubsetReset
+  } = props;
+
+  return (
+    <ButtonGroup style={{ marginRight: "10px" }}>
+      <Tooltip
+        content="Subset to currently selected cells and associated metadata"
+        position="bottom"
+        hoverOpenDelay={globals.tooltipHoverOpenDelay}
+      >
+        <AnchorButton
+          data-testid="subset-button"
+          disabled={!subsetPossible}
+          icon="pie-chart"
+          onClick={handleSubset}
+        />
+      </Tooltip>
+      <Tooltip
+        content="Undo subset and show all cells and associated metadata"
+        position="bottom"
+        hoverOpenDelay={globals.tooltipHoverOpenDelay}
+      >
+        <AnchorButton
+          data-testid="reset-subset-button"
+          disabled={!subsetResetPossible}
+          icon="full-circle"
+          onClick={handleSubsetReset}
+        />
+      </Tooltip>
+    </ButtonGroup>
+  );
+}
+
+export default Subset;

client/src/components/menubar/undoRedo.js

@@ -5,11 +5,8 @@ import { tooltipHoverOpenDelay } from "../../globals";
 
 function InformationMenu(props) {
   const {
-    resettingInterface,
     undoDisabled,
     redoDisabled,
-    resetInterface,
-    isResetDisabled,
     dispatch
   } = props;
   return (
@@ -29,6 +26,7 @@ function InformationMenu(props) {
           style={{
             cursor: "pointer"
           }}
+          data-testid="undo"
         />
       </Tooltip>
       <Tooltip
@@ -46,23 +44,7 @@ function InformationMenu(props) {
           style={{
             cursor: "pointer"
           }}
-        />
+          data-testid="redo"
-      </Tooltip>
-      <Tooltip
-        content="Reset cellxgene, clearing all selections"
-        position="bottom"
-        hoverOpenDelay={tooltipHoverOpenDelay}
-      >
-        <AnchorButton
-          disabled={isResetDisabled()}
-          style={{ marginLeft: 10 }}
-          type="button"
-          loading={resettingInterface}
-          intent="none"
-          icon="refresh"
-          onClick={resetInterface}
-          data-testid="reset"
-          data-testclass={`resetting-${resettingInterface}`}
         />
       </Tooltip>
     </div>

client/src/globals.js

@@ -1,7 +1,8 @@
 import { Colors } from "@blueprintjs/core";
+import { dispatchNetworkErrorMessageToUser } from "./util/actionHelpers";
 
 /* if a categorical metadata field has more options than this, truncate */
-export const maxCategoricalOptionsToDisplay = 100;
+export const maxCategoricalOptionsToDisplay = 200;
 
 /* default "unassigned" value for user-created categorical metadata */
 export const unassignedCategoryLabel = "unassigned";
@@ -72,18 +73,32 @@ export const leftSidebarSectionHeading = {
 export const leftSidebarSectionPadding = 10;
 export const categoryLabelDisplayStringLongLength = 27;
 export const categoryLabelDisplayStringShortLength = 11;
+export const categoryDisplayStringMaxLength = 27;
+
+export const maxUserDefinedGenes = 25;
+export const maxGenes = 100;
 
 /* various timing-related behaviors */
 export const tooltipHoverOpenDelay = 1000; /* ms delay before a tooltip displays */
 export const tooltipHoverOpenDelayQuick = 500;
 
-let _API = {
+let _API;
-  // prefix: "http://api.clustering.czi.technology/api/",
+
-  // prefix: "http://tabulamuris.cxg.czi.technology/api/",
+if (window.CELLXGENE && window.CELLXGENE.API) {
-  // prefix: "http://api-staging.clustering.czi.technology/api/",
+  _API = window.CELLXGENE.API;
-  prefix: "http://localhost:5005/api/",
+} else {
-  version: "v0.2/"
+  if (process.env.CXG_SERVER_PORT === undefined) {
-};
+    const errorMessage = "Please set the CXG_SERVER_PORT environment variable.";
+    dispatchNetworkErrorMessageToUser(errorMessage);
+    throw new Error(errorMessage);
+  }
+  _API = {
+    // prefix: "http://api.clustering.czi.technology/api/",
+    // prefix: "http://tabulamuris.cxg.czi.technology/api/",
+    // prefix: "http://api-staging.clustering.czi.technology/api/",
+    prefix: `http://localhost:${process.env.CXG_SERVER_PORT}/api/`,
+    version: "v0.2/"
+  };
+}
 
-if (window.CELLXGENE && window.CELLXGENE.API) _API = window.CELLXGENE.API;
 export const API = _API;

client/src/reducers/annotations.js

@@ -55,61 +55,40 @@ const Annotations = (
     }
 
     /* CATEGORY */
-    case "annotation: activate add new label mode":
+    case "annotation: activate add new label mode": {
       return {
         ...state,
         isAddingNewLabel: true,
         categoryAddingNewLabel: action.data
       };
-    case "annotation: activate add new ontology label mode":
+    }
-      return {
-        ...state,
-        isAddingNewLabelFromOntology: true,
-        categoryAddingNewLabelFromOntology: action.data
-      };
-    case "annotation: disable add new ontology label mode":
-      return {
-        ...state,
-        isAddingNewLabelFromOntology: false,
-        categoryAddingNewLabelFromOntology: null
-      };
 
-    case "annotation: disable add new label mode":
+    case "annotation: disable add new label mode": {
       return {
         ...state,
         isAddingNewLabel: false,
         categoryAddingNewLabel: null
       };
-    /* this handles both cases, since at this point it's just a string */
+    }
-    case "annotation: add new label to category":
+
-      return {
+    case "annotation: activate category edit mode": {
-        ...state,
-        isAddingNewLabel: false,
-        categoryAddingNewLabel: null,
-        isAddingNewLabelFromOntology: false,
-        categoryAddingNewLabelFromOntology: null
-      };
-    case "annotation: activate category edit mode":
       return {
         ...state,
         isEditingCategoryName: true,
         categoryBeingEdited: action.data
       };
-    case "annotation: disable category edit mode":
+    }
-      return {
+
-        ...state,
+    case "annotation: disable category edit mode": {
-        isEditingCategoryName: false,
-        categoryBeingEdited: null
-      };
-    case "annotation: category edited":
       return {
         ...state,
         isEditingCategoryName: false,
         categoryBeingEdited: null
       };
+    }
 
     /* LABEL */
-    case "annotation: activate edit label mode":
+    case "annotation: activate edit label mode": {
       return {
         ...state,
         isEditingLabelName: true,
@@ -118,18 +97,16 @@ const Annotations = (
           label: action.categoryIndex
         }
       };
-    case "annotation: cancel edit label mode":
+    }
-      return {
+
-        ...state,
+    case "annotation: cancel edit label mode": {
-        isEditingLabelName: false,
-        labelEditable: { category: null, label: null }
-      };
-    case "annotation: label edited":
       return {
         ...state,
         isEditingLabelName: false,
         labelEditable: { category: null, label: null }
       };
+    }
+
     default:
       return state;
   }

client/src/reducers/autosave.js

@@ -2,7 +2,8 @@ const Autosave = (
 	state = {
 		saveInProgress: false,
 		error: false,
-		lastSavedObsAnnotations: null
+		lastSavedObsAnnotations: null,
+		initialDataLoadComplete: false
 	},
 	action,
 	nextSharedState
@@ -15,7 +16,8 @@ const Autosave = (
 				...state,
 				error: false,
 				saveInProgress: false,
-				lastSavedObsAnnotations: universe.obsAnnotations
+				lastSavedObsAnnotations: universe.obsAnnotations,
+				initialDataLoadComplete: true,
 			};
 		}
 

client/src/reducers/cascade.js

@@ -1,6 +1,6 @@
 export default function cascadeReducers(arg) {
   /*
-  Combined a set of cascading reducers into a single reducer.  Cascading
+  Combine a set of cascading reducers into a single reducer.  Cascading
   reducers are reducers which may rely on state computed by another reducer.
   Therefore, they:
   - must be composed in a particular order (currently, this is a simple
@@ -13,7 +13,7 @@ export default function cascadeReducers(arg) {
   - an array of tuples, [ [key1, reducer1], [key2, reducer2], ... ]
   Ie,  cascadeReducers([ ["a", reduceA], ["b", reduceB] ])
 
-  Each reducer will be called with the sigature:
+  Each reducer will be called with the signature:
       (prevState, action, sharedNextState, sharedPrevState) => newState
 
   cascadeReducers will build a composite newState object, much

client/src/reducers/categoricalSelection.js

@@ -1,12 +1,4 @@
 import { ControlsHelpers as CH } from "../util/stateManager";
-import * as globals from "../globals";
-
-function maxCategoryItems(state) {
-  return (
-    state.config.parameters?.["max-category-items"] ??
-    globals.configDefaults.parameters["max-category-items"]
-  );
-}
 
 const CategoricalSelection = (
   state,
@@ -22,11 +14,32 @@ const CategoricalSelection = (
       const { world } = nextSharedState;
       const newState = CH.createCategoricalSelection(
         world,
-        CH.selectableCategoryNames(world, maxCategoryItems(prevSharedState))
+        CH.selectableCategoryNames(
+          world.schema,
+          CH.maxCategoryItems(prevSharedState.config)
+        )
       );
       return newState;
     }
 
+    case "universe: column load success": {
+      const { dim } = action;
+      if (dim !== "obsAnnotations") return state;
+
+      const { dataframe } = action;
+      const { world } = nextSharedState;
+      const names = CH.selectableCategoryNames(
+        world.schema,
+        CH.maxCategoryItems(prevSharedState.config),
+        dataframe.colIndex.keys()
+      );
+      if (names.length === 0) return state;
+      return {
+        ...state,
+        ...CH.createCategoricalSelection(world, names)
+      };
+    }
+
     case "categorical metadata filter select": {
       /*
       Set the specific category in this field to false

client/src/reducers/colors.js

@@ -12,12 +12,12 @@ const ColorsReducer = (
   prevSharedState
 ) => {
   switch (action.type) {
-    case "initial data load complete (universe exists)":
+    case "universe exists, but loading is still in progress": {
-    case "reset World to eq Universe": {
+      /* initialize everything with default colors, no mode, no color-by accessor */
       const { world } = nextSharedState;
       const colorMode = null;
       const colorAccessor = null;
-      const { rgb, scale } = ColorHelpers.createColors(world, colorMode);
+      const { rgb, scale } = ColorHelpers.createColors(world);
       return {
         ...state,
         colorAccessor,
@@ -27,6 +27,22 @@ const ColorsReducer = (
       };
     }
 
+    case "reset World to eq Universe": {
+      /* need to rebuild colors as world may have changed, but don't switch modes */
+      const { world } = nextSharedState;
+      const { colorMode, colorAccessor } = state;
+      const { rgb, scale } = ColorHelpers.createColors(
+        world,
+        colorMode,
+        colorAccessor
+      );
+      return {
+        ...state,
+        rgb,
+        scale
+      };
+    }
+
     case "set clip quantiles":
     case "set World to current selection": {
       const { world: prevWorld, controls: prevControls } = prevSharedState;

client/src/reducers/controls.js

@@ -1,6 +1,7 @@
 // jshint esversion: 6
 
 import _ from "lodash";
+import { subsetAndResetGeneLists } from "../util/stateManager/controlsHelpers";
 
 const Controls = (
   state = {
@@ -41,8 +42,31 @@ const Controls = (
     case "initial data load start": {
       return { ...state, loading: true };
     }
+    case "universe: column load success": {
+      /* 
+      we are in a loading state until the following are true:
+        * universe exists (if partially)
+        * embeddings (obsLayout) are loaded
+        * varAnnotations index is loaded
+      */
+      const universeExists = !!nextSharedState.universe;
+      const embeddingsExist =
+        !nextSharedState.universe?.obsLayout?.isEmpty() ?? false;
+      const varIndex = nextSharedState.universe.schema.annotations.var.index;
+      const varAnnotationsIndexExists =
+        universeExists &&
+        nextSharedState.universe.varAnnotations.hasCol(varIndex);
+      return {
+        ...state,
+        loading: !(
+          universeExists &&
+          embeddingsExist &&
+          varAnnotationsIndexExists
+        )
+      };
+    }
     case "initial data load complete (universe exists)": {
-      /* first light - create world & other data-driven defaults */
+      /* now fully loaded */
       return {
         ...state,
         loading: false,
@@ -51,16 +75,22 @@ const Controls = (
       };
     }
     case "reset World to eq Universe": {
+      const [ newUserDefinedGenes, newDiffExpGenes ] = subsetAndResetGeneLists(state);
       return {
         ...state,
-        resettingInterface: false
+        resettingInterface: false,
+        userDefinedGenes: newUserDefinedGenes,
+        diffexpGenes: newDiffExpGenes
       };
     }
     case "set World to current selection": {
+      const [ newUserDefinedGenes, newDiffExpGenes ] = subsetAndResetGeneLists(state);
       return {
         ...state,
         loading: false,
-        error: null
+        error: null,
+        userDefinedGenes: newUserDefinedGenes,
+        diffexpGenes: newDiffExpGenes
       };
     }
     case "request user defined gene started": {

client/src/reducers/crossfilter.js

@@ -23,13 +23,37 @@ const CrossfilterReducerBase = (
   prevSharedState
 ) => {
   switch (action.type) {
-    case "initial data load complete (universe exists)": {
+    case "universe: column load success": {
       const { world, layoutChoice } = nextSharedState;
-      const crossfilter = World.createObsDimensions(
+      const { obsAnnotations, obsLayout } = world;
-        new Crossfilter(world.obsAnnotations),
+
-        world,
+      // ignore var dimension loads as these are not currently selectable
-        layoutChoice.currentDimNames
+      if (action.dim === "varAnnotations") return state;
-      );
+
+      /* 
+      during bootstrap loading, we don't know if obsLayout or obsAnnotations
+      will load first. Take whichever arrives and is not empty (so that our
+      crossfilter has the right dimensionality).
+      */
+      let crossfilter =
+        state ??
+        new Crossfilter(obsAnnotations.isEmpty() ? obsLayout : obsAnnotations);
+
+      // add layout dimension, if not already present
+      if (
+        obsLayout.hasCol(layoutChoice.currentDimNames[0]) &&
+        !crossfilter.hasDimension(XYDimName)
+      ) {
+        crossfilter = crossfilter.addDimension(
+          XYDimName,
+          "spatial",
+          obsLayout.col(layoutChoice.currentDimNames[0]).asArray(),
+          obsLayout.col(layoutChoice.currentDimNames[1]).asArray()
+        );
+      }
+
+      // add any missing obsAnnotations
+      crossfilter = World.addObsDimensions(crossfilter, world);
       return crossfilter;
     }
 
@@ -278,10 +302,19 @@ const CrossfilterReducer = (
     nextSharedState,
     prevSharedState
   );
-  if (!nextState || nextState.all() === nextSharedState.world.obsAnnotations) {
+  /*
+  update the data in the crossfilter to point at the current obsAnnotations, IF
+  they are not empty.  If empty, leave it alone (can occur during boostrap loading).
+  */
+  const nextObsAnnotations = nextSharedState.world?.obsAnnotations;
+  if (
+    !nextState ||
+    nextState.all() === nextObsAnnotations ||
+    nextObsAnnotations.isEmpty()
+  ) {
     return nextState;
   }
-  return nextState.setData(nextSharedState.world.obsAnnotations);
+  return nextState.setData(nextObsAnnotations);
 };
 
 export default CrossfilterReducer;

client/src/reducers/index.js

@@ -21,6 +21,7 @@ import autosave from "./autosave";
 import ontology from "./ontology";
 import centroidLabels from "./centroidLabels";
 import pointDialation from "./pointDilation";
+import { reembedController, reembedding } from "./reembed";
 
 import undoableConfig from "./undoableConfig";
 
@@ -31,6 +32,7 @@ const Reducer = undoable(
     ["world", world],
     ["ontology", ontology],
     ["annotations", annotations],
+    ["reembedding", reembedding],
     ["layoutChoice", layoutChoice],
     ["categoricalSelection", categoricalSelection],
     ["continuousSelection", continuousSelection],
@@ -42,6 +44,7 @@ const Reducer = undoable(
     ["responsive", responsive],
     ["centroidLabels", centroidLabels],
     ["pointDilation", pointDialation],
+    ["reembedController", reembedController],
     ["autosave", autosave],
     ["resetCache", resetCache]
   ]),
@@ -57,7 +60,8 @@ const Reducer = undoable(
     "differential",
     "layoutChoice",
     "centroidLabels",
-    "annotations"
+    "annotations",
+    "reembedding"
   ],
   undoableConfig
 );

client/src/reducers/layoutChoice.js

@@ -14,6 +14,14 @@ function bestDefaultLayout(layouts) {
   return layouts[0];
 }
 
+function setToDefaultLayout(world) {
+  const { schema } = world;
+  const available = schema.layout.obs.map(v => v.name).sort();
+  const current = bestDefaultLayout(available);
+  const currentDimNames = schema.layout.obsByName[current].dims;
+  return { available, current, currentDimNames };
+}
+
 const LayoutChoice = (
   state = {
     available: [], // all available choices
@@ -24,14 +32,13 @@ const LayoutChoice = (
   nextSharedState
 ) => {
   switch (action.type) {
-    case "initial data load complete (universe exists)":
+    case "universe exists, but loading is still in progress": {
-    case "reset World to eq Universe": {
       // set default to default
-      const { schema } = nextSharedState.world;
+      const { universe } = nextSharedState;
-      const available = schema.layout.obs.map(v => v.name).sort();
+      return {
-      const current = bestDefaultLayout(available);
+        ...state,
-      const currentDimNames = schema.layout.obsByName[current].dims;
+        ...setToDefaultLayout(universe)
-      return { available, current, currentDimNames };
+      };
     }
 
     case "set layout choice": {
@@ -41,6 +48,31 @@ const LayoutChoice = (
       return { ...state, current, currentDimNames };
     }
 
+    case "reembed: add reembedding": {
+      const name = action.schema.name;
+      const available = Array.from(new Set(state.available).add(name));
+      return {
+        ...state,
+        available
+      };
+    }
+
+    case "reembed: clear all reembeddings": {
+      const { universe } = nextSharedState;
+      const { current } = state;
+      const dflt = setToDefaultLayout(universe);
+      if (dflt.available.includes(current)) {
+        return {
+          ...state,
+          available: dflt.available
+        };
+      }
+      return {
+        ...state,
+        ...dflt
+      };
+    }
+
     default: {
       return state;
     }

client/src/reducers/ontology.js

@@ -1,4 +1,3 @@
-// jshint esversion: 6
 const Ontology = (
   state = {
     enabled: false, // are ontology terms enabled?
@@ -9,10 +8,12 @@ const Ontology = (
   action
 ) => {
   switch (action.type) {
-    case "configuration load complete":
+    case "configuration load complete": {
+      /* eslint-disable camelcase */
       const enabled =
         action.config?.parameters?.annotations_cell_ontology_enabled ?? false;
       const terms = action.config?.parameters?.annotations_cell_ontology_terms;
+      /* eslint-enable camelcase */
       const termSet = new Set(terms);
       return {
         ...state,
@@ -21,8 +22,10 @@ const Ontology = (
         terms,
         termSet
       };
-    default:
+    }
+    default: {
       return state;
+    }
   }
 };
 

client/src/reducers/reembed.js

@@ -0,0 +1,64 @@
+/*
+controller state is not part of the undo/redo history
+*/
+export const reembedController = (
+  state = {
+    pendingFetch: null
+  },
+  action
+) => {
+  switch (action.type) {
+    case "reembed: request start": {
+      return {
+        ...state,
+        pendingFetch: action.abortableFetch
+      };
+    }
+    case "reembed: request aborted":
+    case "reembed: request cancel":
+    case "reembed: request completed": {
+      return {
+        ...state,
+        pendingFetch: null
+      };
+    }
+    default: {
+      return state;
+    }
+  }
+};
+
+/*
+actual reembedding data is part of the undo/redo history
+*/
+export const reembedding = (
+  state = {
+    reembeddings: new Map()
+  },
+  action
+) => {
+  switch (action.type) {
+    case "reembed: add reembedding": {
+      const { schema, embedding } = action;
+      const { name } = schema.name;
+      const { reembeddings } = state;
+      return {
+        ...state,
+        reembeddings: new Map(reembeddings).set(name, {
+          name,
+          schema,
+          embedding
+        })
+      };
+    }
+    case "reembed: clear all reembeddings": {
+      return {
+        ...state,
+        reembeddings: new Map()
+      };
+    }
+    default: {
+      return state;
+    }
+  }
+};

client/src/reducers/undoable.js

@@ -85,7 +85,6 @@ const Undoable = (reducer, undoableKeys, options = {}) => {
     );
     const newPast = [...past];
     const newState = newPast.pop();
-    const newStateFilterState = newState[filterStateKey];
     const newFuture = push(future, currentUndoableState);
     const nextState = {
       ...currentState,
@@ -94,7 +93,6 @@ const Undoable = (reducer, undoableKeys, options = {}) => {
       [futureKey]: newFuture,
       [pendingKey]: null
     };
-    nextState[filterStateKey] = newStateFilterState;
     return nextState;
   }
 
@@ -139,12 +137,13 @@ const Undoable = (reducer, undoableKeys, options = {}) => {
   */
   function skip(currentState, action, filterState) {
     const past = currentState[pastKey] || [];
+    const future = currentState[futureKey] || [];
     const pending = currentState[pendingKey];
     const res = reducer(currentState, action);
     return {
       ...res,
       [pastKey]: past,
-      [futureKey]: [],
+      [futureKey]: future,
       [filterStateKey]: filterState,
       [pendingKey]: pending
     };
@@ -218,18 +217,19 @@ const Undoable = (reducer, undoableKeys, options = {}) => {
     },
     action
   ) => {
+    if (debug > 1) console.log("---- ACTION", action.type);
     const aType = action.type;
     switch (aType) {
       case "@@undoable/undo": {
-        return undo(currentState, action);
+        return undo(currentState);
       }
 
       case "@@undoable/redo": {
-        return redo(currentState, action);
+        return redo(currentState);
       }
 
       case "@@undoable/clear": {
-        return clear(currentState, action);
+        return clear(currentState);
       }
 
       default: {

client/src/reducers/undoableConfig.js

@@ -11,10 +11,11 @@ const skipOnActions = new Set([
   "url changed",
   "interface reset started",
   "initial data load start",
+  "universe: column load success",
+  "universe exists, but loading is still in progress",
   "configuration load complete",
   "increment graph render counter",
   "window resize",
-  "user reset start",
   "reset colorscale",
   "reset centroid labels",
 
@@ -33,18 +34,28 @@ const skipOnActions = new Set([
   "get single gene expression for coloring error",
 
   "category value mouse hover start",
-  "category value mouse hover end"
+  "category value mouse hover end",
+
+  /* autosave annotations */
+  "writable obs annotations - save complete",
+  "writable obs annotations - save started",
+  "writable obs annotations - save error",
+
+  /* annotation component action */
+  "annotation: activate add new label mode",
+  "annotation: disable add new label mode",
+  "annotation: activate category edit mode",
+  "annotation: disable category edit mode",
+  "annotation: activate edit label mode",
+  "annotation: cancel edit label mode",
+  "set annotations collection name"
 ]);
 
 /*
 identical, repeated occurances of these action types will be debounced.
 Entire action must be identical (all keys).
 */
-const debounceOnActions = new Set([
+const debounceOnActions = new Set([]);
-  "color by categorical metadata",
-  "color by continuous metadata",
-  "color by expression"
-]);
 
 /*
 history will be cleared when these actions occur
@@ -53,7 +64,6 @@ const clearOnActions = new Set([
   "initial data load complete (universe exists)",
   "reset World to eq Universe",
   "initial data load error",
-  "user reset end"
 ]);
 
 /*
@@ -89,7 +99,8 @@ const saveOnActions = new Set([
   "annotation: delete category",
   "annotation: label edited",
   "annotation: label current cell selection",
-  "annotation: delete label"
+  "annotation: delete label",
+  "annotation: category edited"
 ]);
 
 /**
@@ -134,7 +145,7 @@ Signature:  (fsm, event, from) => undoableAction
 const onFsmError = (fsm, event, from) => {
   console.error(`FSM error [event: "${event}", state: "${from}"]`, fsm);
   // In production, try to recover gracefully if we have unexpected state
-  return clear(fsm);
+  return clear();
 };
 
 /*
@@ -170,8 +181,6 @@ const actionFilter = debug => (state, action, prevFilterState) => {
   }
   if (
     debounceOnActions.has(actionType) &&
-    prevFilterState !== undefined &&
-    prevFilterState.prevAction !== undefined &&
     shallowObjectEq(action, prevFilterState.prevAction)
   ) {
     return { [actionKey]: "skip", [stateKey]: filterState };
@@ -227,7 +236,12 @@ function shallowArrayEq(arrA, arrB) {
 }
 
 /* configuration for the undoable meta reducer */
-const debug = false; // set truish for undoble debugging
+/*
+debug: set to any falsish value to disable logging of helpful debugging information.
+Set to true or 1 for base logging, high number for more verbosity (currently only 1/true
+or 2).
+*/
+const debug = false;
 const undoableConfig = {
   debug,
   historyLimit: 50, // maximum history size

client/src/reducers/universe.js

@@ -1,4 +1,9 @@
 import { unassignedCategoryLabel } from "../globals";
+import {
+  addObsAnnotations,
+  addVarAnnotations,
+  addObsLayout
+} from "../util/stateManager/universe";
 import {
   World,
   ControlsHelpers,
@@ -7,11 +12,38 @@ import {
 
 const Universe = (state = null, action, nextSharedState, prevSharedState) => {
   switch (action.type) {
-    case "initial data load complete (universe exists)": {
+    case "universe exists, but loading is still in progress": {
       const { universe } = action;
       return universe;
     }
 
+    case "universe: column load success": {
+      const { dim, dataframe } = action;
+      switch (dim) {
+        case "obsAnnotations": {
+          return {
+            ...state,
+            ...addObsAnnotations(state, dataframe)
+          };
+        }
+        case "varAnnotations": {
+          return {
+            ...state,
+            ...addVarAnnotations(state, dataframe)
+          };
+        }
+        case "obsLayout": {
+          return {
+            ...state,
+            ...addObsLayout(state, dataframe)
+          };
+        }
+        default: {
+          throw new Error("action handler not implemented");
+        }
+      }
+    }
+
     case "expression load success": {
       let { varData } = state;
 
@@ -30,9 +62,7 @@ const Universe = (state = null, action, nextSharedState, prevSharedState) => {
       const { userDefinedGenes, diffexpGenes } = prevSharedState;
       const allTheGenesWeNeed = [
         ...new Set(
-          userDefinedGenes,
+          [userDefinedGenes, diffexpGenes, Object.keys(action.expressionData)].filter(ele => ele).flat()
-          diffexpGenes,
-          Object.keys(action.expressionData)
         )
       ];
       varData = ControlsHelpers.pruneVarDataCache(varData, allTheGenesWeNeed);

client/src/reducers/world.js

@@ -4,6 +4,10 @@ import {
   ControlsHelpers,
   AnnotationsHelpers
 } from "../util/stateManager";
+import {
+  addObsLayout,
+  removeObsLayout
+} from "../util/stateManager/schemaHelpers";
 import clip from "../util/clip";
 import quantile from "../util/quantile";
 
@@ -24,13 +28,28 @@ const WorldReducer = (
   prevSharedState
 ) => {
   switch (action.type) {
-    case "initial data load complete (universe exists)":
+    case "universe exists, but loading is still in progress":
     case "reset World to eq Universe": {
       const { universe } = nextSharedState;
       const world = World.createWorldFromEntireUniverse(universe);
       return world;
     }
 
+    case "universe: column load success": {
+      /* incremental initial data load - always assumes world == universe */
+      const { universe } = nextSharedState;
+      const { dim } = action;
+      return {
+        ...state,
+        schema: universe.schema,
+        [dim]: universe[dim].clone(),
+        unclipped: {
+          ...state.unclipped,
+          [dim]: universe[dim].clone()
+        }
+      };
+    }
+
     case "set World to current selection": {
       /* Set viewable world to be the currently selected data */
       const world = World.createWorldBySelection(
@@ -87,15 +106,10 @@ const WorldReducer = (
       const { userDefinedGenes, diffexpGenes } = prevSharedState;
       const allTheGenesWeNeed = [
         ...new Set(
-          userDefinedGenes,
+          [userDefinedGenes, diffexpGenes, Object.keys(action.expressionData)].filter(ele => ele).flat()
-          diffexpGenes,
-          Object.keys(action.expressionData)
         )
       ];
-      unclippedVarData = ControlsHelpers.pruneVarDataCache(
+      unclippedVarData = ControlsHelpers.pruneVarDataCache(unclippedVarData, allTheGenesWeNeed);
-        unclippedVarData,
-        allTheGenesWeNeed
-      );
 
       // at this point, we have the unclipped data in unclippedVarData.
       // Now create clipped.
@@ -267,6 +281,52 @@ const WorldReducer = (
       };
     }
 
+    case "reembed: add reembedding": {
+      // new embedding loaded, which *only* affects world's layout.
+      // It may be new, or it may replace a previous re-embedding.
+      const { obsLayout: origObsLayout, schema: origSchema } = state;
+      const { embedding, schema: embeddingSchema } = action;
+
+      const { dims, name } = embeddingSchema;
+      let obsLayout = origObsLayout;
+      let schema = origSchema;
+
+      // alias the names the server sent us, in case they were not the same as the schema
+      const embedingLabels = embedding.colIndex.keys();
+      const labels = {
+        [embedingLabels[0]]: dims[0],
+        [embedingLabels[1]]: dims[1]
+      };
+      obsLayout = obsLayout.withColsFrom(embedding, labels);
+      schema = addObsLayout(schema, embeddingSchema);
+      return {
+        ...state,
+        obsLayout,
+        schema
+      };
+    }
+
+    case "reembed: clear all reembeddings": {
+      // reembedding was cleared -- remove from layout
+      const { obsLayout: origObsLayout, schema: origSchema } = state;
+      const { reembedding } = prevSharedState;
+
+      let schema = origSchema;
+      let obsLayout = origObsLayout;
+
+      reembedding.reembeddings.forEach((emb, name) => {
+        const { dims } = emb.schema;
+        obsLayout = obsLayout.dropCol(dims[0]);
+        obsLayout = obsLayout.dropCol(dims[1]);
+        schema = removeObsLayout(schema, name);
+      });
+      return {
+        ...state,
+        obsLayout,
+        schema
+      };
+    }
+
     default: {
       return state;
     }

client/src/util/catLabelSort.js

@@ -10,7 +10,6 @@ TL;DR: sort order is:
 
 import isNumber from "is-number";
 import * as globals from "../globals";
-import { memoize } from "./dataframe/util";
 
 function caseInsensitiveCompare(a, b) {
   const textA = String(a).toUpperCase();

client/src/util/centroid.js

@@ -4,13 +4,96 @@ import { unassignedCategoryLabel } from "../globals";
 
 /*
   Centroid coordinate calculation
+  Calculates centroids for displaying
+  In the case that a category is truncated, the truncated labels will not have
+    centroids calculated
 */
 
-/* 
+/*
-  calcMedianCentroid goes through a given metadata category
+Generates a mapping of categorical values to data needed to calculate centroids
-  fetches each cell's coordinates grouping by category value.
+categoricalValue -> {
+ length: int,
+ holdsFinite: Boolean,
+ xCoordinates: Float32Array,
+ yCoordinates: Float32Array
+}
+*/
+const getCoordinatesByCategoricalValues = (
+  obsAnnotations,
+  obsLayout,
+  categoryName,
+  layoutDimNames,
+  categoricalSelection,
+  schemaObsByName
+) => {
+  const coordsByCategoryLabel = new Map();
 
-  It then calculates the median value and puts that in the array
+  const categoryArray = obsAnnotations.col(categoryName).asArray();
+
+  const layoutXArray = obsLayout.col(layoutDimNames[0]).asArray();
+  const layoutYArray = obsLayout.col(layoutDimNames[1]).asArray();
+
+  const { categoryValueIndices, categoryValueCounts } =
+    categoricalSelection?.[categoryName] || {};
+
+  // If the coloredBy is not a categorical col
+  if (categoryValueIndices === undefined) {
+    return coordsByCategoryLabel;
+  }
+
+  // Check to see if the current category is a user created annotation
+  const isUserAnno = schemaObsByName[categoryName].writable;
+
+  // Iterate over all cells
+  for (let i = 0, len = categoryArray.length; i < len; i += 1) {
+    // Fetch the categorical value of the current cell
+    const categoryValue = categoryArray[i];
+
+    // Get the index of the categoryValue within the category
+    const categoryValueIndex = categoryValueIndices.get(categoryValue);
+
+    // If the category is truncated and this value is removed,
+    //  it will not be assigned a category value and will not be
+    //  labeled on the graph
+    // If the user created this category,
+    //  do not create a label for the `unassigned` value
+    if (
+      categoryValueIndex !== undefined &&
+      !(isUserAnno && categoryValue === unassignedCategoryLabel)
+    ) {
+      // Create/fetch the scratchpad value
+      let coords = coordsByCategoryLabel.get(categoryValue);
+      if (coords === undefined) {
+        // Get the number of cells which are in the categorical value
+        const numInCategoricalValue = categoryValueCounts[categoryValueIndex];
+        coords = {
+          hasFinite: false,
+          xCoordinates: new Float32Array(numInCategoricalValue),
+          yCoordinates: new Float32Array(numInCategoricalValue),
+          length: 0
+        };
+        coordsByCategoryLabel.set(categoryValue, coords);
+      }
+
+      coords.hasFinite =
+        coords.hasFinite ||
+        (Number.isFinite(layoutXArray[i]) && Number.isFinite(layoutYArray[i]));
+
+      const coordinatesLength = coords.length;
+
+      coords.xCoordinates[coordinatesLength] = layoutXArray[i];
+      coords.yCoordinates[coordinatesLength] = layoutYArray[i];
+
+      coords.length = coordinatesLength + 1;
+    }
+  }
+  return coordsByCategoryLabel;
+};
+
+/* 
+  calcMedianCentroid calculates the median coordinates for categorical values in a given metadata field 
+
+  categoricalValue -> [x-Coordinate, y-Coordinate]
 */
 
 const calcMedianCentroid = (
@@ -21,85 +104,39 @@ const calcMedianCentroid = (
   categoricalSelection,
   schemaObsByName
 ) => {
-  const categoryArray = obsAnnotations.col(categoryName).asArray();
+  // generate a map describing the coordinates for each value within the given category
+  const dataMap = getCoordinatesByCategoricalValues(
+    obsAnnotations,
+    obsLayout,
+    categoryName,
+    layoutDimNames,
+    categoricalSelection,
+    schemaObsByName
+  );
 
-  const layoutXArray = obsLayout.col(layoutDimNames[0]).asArray();
+  // categoricalValue => [medianXCoordinate, medianYCoordinate]
-  const layoutYArray = obsLayout.col(layoutDimNames[1]).asArray();
   const coordinates = new Map();
 
-  // Iterate over all the cells in the category
-  for (let i = 0, len = categoryArray.length; i < len; i += 1) {
-    const categoryValue = categoryArray[i];
-
-    // Get the index of the categoryValue within the category
-    // If the category is truncated and this value is removed,
-    // it will not be assigned a category value and will not be
-    // labeled on the graph
-    const categoryValueIndex = categoricalSelection[
-      categoryName
-    ].categoryValueIndices.get(categoryValue);
-
-    // Check to see if the current category is a user created annotation
-    // if the user created this category, do not create a label for the `unassigned` value
-    const isUserAnno = schemaObsByName[categoryName].writable;
-
-    if (
-      categoryValueIndex !== undefined &&
-      !(isUserAnno && categoryValue === unassignedCategoryLabel)
-    ) {
-      // Get the number of cells which are in the category value
-      const numInCategoryValue =
-        categoricalSelection[categoryName].categoryValueCounts[
-          categoryValueIndex
-        ];
-
-      // Create/fetch the valueArray,
-      // which is what the key points to in the `coordinates` hashmap
-      const valueArray = coordinates.get(categoryValue) || [
-        false, // hasFinite
-        0, // index
-        new Float32Array(numInCategoryValue), // x coordinates
-        new Float32Array(numInCategoryValue) // y coordinates
-      ];
-      const index = valueArray[1];
-      let hasFinite = valueArray[0];
-
-      hasFinite =
-        hasFinite ||
-        (Number.isFinite(layoutXArray[i]) && Number.isFinite(layoutYArray[i]));
-
-      valueArray[0] = hasFinite;
-      valueArray[1] = index + 1;
-      valueArray[2][index] = layoutXArray[i];
-      valueArray[3][index] = layoutYArray[i];
-
-      coordinates.set(categoryValue, valueArray);
-    }
-  }
-
   // Iterate over the recently created map
-  coordinates.forEach((value, key) => {
+  dataMap.forEach((value, key) => {
-    // If there are coordinates for this cateogrical value,
+    // If there are coordinates for this categorical value,
     // and there is a finite coordinate for the category value
-    if (value[2].length > 0 && value[3].length > 0 && value[0]) {
+    if (value.length > 0 && value.hasFinite) {
-      // Find the median x and y coordinate
+      const calculatedCoordinates = [];
-      // and insert them into the first two indices
+
-      value[0] = quantile([0.5], value[2])[0];
+      // Find and store the median x and y coordinate
-      value[1] = quantile([0.5], value[3])[0];
+      calculatedCoordinates[0] = quantile([0.5], value.xCoordinates)[0];
-      // Remove the last two elements (where the arrays of coordinates were)
+      calculatedCoordinates[1] = quantile([0.5], value.yCoordinates)[0];
-      value.pop();
+
-      value.pop();
+      coordinates.set(key, calculatedCoordinates);
-    } else {
-      // remove the entry if not
-      coordinates.delete(key);
     }
   });
+
   // return the map: categoricalValue -> [medianXCoordinate, medianYCoordinate]
   return coordinates;
 };
 
 // A simple function to hash the parameters
-// (not 100% on world hash, Bruce will have to check this one out)
 const hashMedianCentroid = (
   obsAnnotations,
   obsLayout,
@@ -114,5 +151,5 @@ const hashMedianCentroid = (
     categorySelection
   )}:${Object.keys(schemaObsByName)}`;
 };
-// export the mmemoized calculation function
+// export the memoized calculation function
 export default memoize(calcMedianCentroid, hashMedianCentroid);

client/src/util/dataframe/dataframe.js

@@ -348,22 +348,90 @@ class Dataframe {
     );
   }
 
-  withColsFrom(dataframe) {
+  withColsFrom(dataframe, labels) {
     /*
     return a new dataframe containing all columns from both `this` and the
-    provided dataframe.
+    provided dataframe argument.
 
-    The row index from `this` will be used.  Both dataframes must have identical
+    The row index from `this` will be used.  All dataframes must have identical
     dimensionality, and no overlapping columns labels.
+
+    Special case, if either dataframe is empty, the other is returned unchanged.
+
+    Arguments:
+    * dataframe: a dataframe to combine with `this`
+    * labels: columns to pull from `dataframe` and combine with `this`.  If falsey,
+      all columns are used.  If an array, must contain a list of labels.  If an
+      Object or Map, the key is the columns to pull, which will be stored into the
+      new dataframe as the value.
+
+    Example:
+
+    newDf = df.withColsFrom(otherDf);         // combines all columns from both
+    newDf = df.withColsFrom(otherDf, ['a']);  // combines df with otherDf['a']
+    newDf = df.withColsFrom(otherDf, {a: 'b'}); // combines df with otherDf['a'], but calls it 'b'
+
      */
-    const dims = [this.dims[0], this.dims[1] + dataframe.dims[1]];
+
+    // resolve the source and dest label names.
+    let srcLabels;
+    let dstLabels;
+    if (!labels) {
+      // combine all columns
+      dstLabels = dataframe.colIndex.keys();
+      srcLabels = dstLabels;
+    } else if (Array.isArray(labels)) {
+      // combine subset of keys with no aliasing
+      dstLabels = labels;
+      srcLabels = labels;
+    } else if (labels instanceof Map) {
+      // aliasing with a Map
+      srcLabels = Array.from(labels.keys());
+      dstLabels = Array.from(labels.values());
+    } else {
+      // aliasing with an Object
+      srcLabels = Object.keys(labels);
+      dstLabels = Object.values(labels);
+    }
+
+    // if datafame is empty, and no specific labels specified, noop.
+    if (dataframe.isEmpty()) {
+      if (!labels || srcLabels.length === 0) return this;
+      throw new Error("Empty dataframe, unable to pick columns");
+    }
+
+    if (this.isEmpty()) {
+      // 1. subset dataframe from source keys
+      // 2. alias names
+      dataframe = dataframe.subset(null, srcLabels);
+      for (let i = 0; i < srcLabels.length; i += 1) {
+        dataframe = dataframe.renameCol(srcLabels[i], dstLabels[i]);
+      }
+      return dataframe;
+    }
+
+    // otherwise, bulid a new dataframe combining columns from both
+
+    const srcOffsets = srcLabels.map(l => dataframe.colIndex.getOffset(l));
+
+    // check for label collisions
+    if (dstLabels.some(this.hasCol, this)) {
+      throw new Error("duplicate key collision");
+    }
+
+    // const dims = [this.dims[0], this.dims[1] + dataframe.dims[1]];
+    const dims = [this.dims[0], this.dims[1] + srcOffsets.length];
     const { rowIndex } = this;
-    const columns = [...this.__columns, ...dataframe.__columns];
+    const columns = [
-    const colIndex = this.colIndex.withLabels(dataframe.colIndex.keys());
+      ...this.__columns,
+      ...srcOffsets.map(i => dataframe.__columns[i])
+    ];
+    const colIndex = this.colIndex.withLabels(dstLabels);
     const columnsAccessor = [
       ...this.__columnsAccessor,
-      ...dataframe.__columnsAccessor
+      ...srcOffsets.map(i => dataframe.__columnsAccessor[i])
     ];
+
     return new this.constructor(
       dims,
       columns,
@@ -373,6 +441,11 @@ class Dataframe {
     );
   }
 
+  withColsFromAll(dataframes = []) {
+    dataframes = Array.isArray(dataframes) ? dataframes : [dataframes];
+    return dataframes.reduce((acc, df) => acc.withColsFrom(df), this);
+  }
+
   dropCol(label) {
     /*
     Create a new dataframe, omitting one columns.
@@ -566,7 +639,7 @@ class Dataframe {
 
     withRowIndex allows assignment of new row index during subset operation.
     If withRowIndex === null, it will reset the index to identity (offset)
-    indexing.  if withRowIndex is a label index object, it will be used
+    indexing. If withRowIndex is a label index object, it will be used
     for the new dataframe.
     */
     return this.__subset(rowOffsets, colOffsets, withRowIndex);

client/src/util/maybeTruncateString.js

@@ -0,0 +1,18 @@
+const maybeTruncateString = (str, maxLength) => {
+
+  let truncatedString = null;
+  if (
+    str.length > maxLength
+  ) {
+    truncatedString = `${str.slice(
+      0,
+      maxLength / 2
+    )}…${str.slice(
+      -maxLength / 2
+    )}`;
+  } 
+
+  return truncatedString || str;
+}
+
+export default maybeTruncateString;

client/src/util/stateManager/annotationsHelpers.js

@@ -1,5 +1,5 @@
 /*
-Helper functions for user-editable nnotations state management.
+Helper functions for user-editable annotations state management.
 See also reducers/annotations.js
 */
 import { unassignedCategoryLabel } from "../../globals";

client/src/util/stateManager/controlsHelpers.js

@@ -11,6 +11,13 @@ import {
   diffexpDimensionName
 } from "../nameCreators";
 
+export function maxCategoryItems(config) {
+  return (
+    config.parameters?.["max-category-items"] ??
+    globals.configDefaults.parameters["max-category-items"]
+  );
+}
+
 /*
 Selection state for categoricals are tracked in an Object that
 has two main components for each category:
@@ -38,7 +45,7 @@ Remember that option values can be ANY js type, except undefined/null.
   }
 */
 function topNCategories(colSchema, summary, N) {
-  /* return top N by occurance in the data, preserving original category order */
+  /* return top N by occurrences in the data, preserving original category order */
   const { categories } = colSchema;
   const counts = categories.map(cat => summary.categoryCounts.get(cat) ?? 0);
 
@@ -51,26 +58,34 @@ function topNCategories(colSchema, summary, N) {
   );
   const topNindices = new Set(sortIndex.slice(0, N));
 
-  const topNCategories = [];
+  const _topNCategories = [];
   const topNCounts = [];
   for (let i = 0; i < categories.length; i += 1) {
     if (topNindices.has(i)) {
-      topNCategories.push(categories[i]);
+      _topNCategories.push(categories[i]);
       topNCounts.push(counts[i]);
     }
   }
-  return [topNCategories, topNCounts];
+  return [_topNCategories, topNCounts];
 }
 
-export function selectableCategoryNames(world, maxCategoryItems) {
+export function selectableCategoryNames(schema, maxCatItems, names) {
-  const { schema } = world;
+  /*
+  return all obs annotation names that are categorical AND have a
+  "reasonably" small number of categories AND are not the index column.
+
+  If the initial name list not provided, use everything in the schema.
+  */
+  if (!schema) return [];
   const { index, columns } = schema.annotations.obs;
+
   return columns
+    .filter(colSchema => !names || names.indexOf(colSchema.name) !== -1)
     .filter(colSchema => {
-      const { name, categories } = colSchema;
+      const { type, name } = colSchema;
-      return (
+      const isSelectableType =
-        categories && categories.length < maxCategoryItems && name !== index
+        type === "string" || type === "boolean" || type === "categorical";
-      );
+      return isSelectableType && name !== index;
     })
     .map(v => v.name);
 }
@@ -157,7 +172,7 @@ export function pruneVarDataCache(varData, needed) {
   */
 
   /*
-  VarDataCacheLowWatermark - this cofig value sets the minimum cache size,
+  VarDataCacheLowWatermark - this config value sets the minimum cache size,
   in columns, below which we don't throw away data.
 
   The value should be high enough so we are caching the maximum which will
@@ -185,3 +200,10 @@ export function pruneVarDataCache(varData, needed) {
   }
   return varData;
 }
+
+export function subsetAndResetGeneLists(state) {
+  const { userDefinedGenes, diffexpGenes } = state;
+  const newUserDefinedGenes = [].concat(userDefinedGenes, diffexpGenes).slice(0, globals.maxGenes);
+  const newDiffExpGenes = [];
+  return [newUserDefinedGenes, newDiffExpGenes];
+}

client/src/util/stateManager/schemaHelpers.js

@@ -1,5 +1,8 @@
 /*
 Helpers for schema management
+
+TODO: all this would be much more natural if done with a framework
+like immutable.js
 */
 import _ from "lodash";
 
@@ -31,17 +34,8 @@ export function indexEntireSchema(schema) {
   return schema;
 }
 
-export function sortAllCategorical(schema) {
+function _copyObsAnno(schema) {
-  /* UI relies on OBS annotation categories being in presentation sort order */
+  /* redux copy conventions - WARNING, only for modifying obs annotations */
-  schema.annotations.obs.columns.forEach(c => {
-    if (c.categories) {
-      c.categories = catLabelSort(c.writable, c.categories);
-    }
-  });
-}
-
-function _copy(schema) {
-  /* redux copy conventions - WARNING, only for modifyign obs annotations */
   return {
     ...schema,
     annotations: {
@@ -51,7 +45,17 @@ function _copy(schema) {
   };
 }
 
-function _reindex(schema) {
+function _copyObsLayout(schema) {
+  return {
+    ...schema,
+    layout: {
+      ...schema.layout,
+      obs: _.cloneDeep(schema.layout.obs)
+    }
+  };
+}
+
+function _reindexObsAnno(schema) {
   /* reindex obs annotations ONLY */
   schema.annotations.obsByName = fromEntries(
     schema.annotations.obs.columns.map(v => [v.name, v])
@@ -59,18 +63,25 @@ function _reindex(schema) {
   return schema;
 }
 
+function _reindexObsLayout(schema) {
+  schema.layout.obsByName = fromEntries(
+    schema.layout.obs.map(v => [v.name, v])
+  );
+  return schema;
+}
+
 export function removeObsAnnoColumn(schema, name) {
-  const newSchema = _copy(schema);
+  const newSchema = _copyObsAnno(schema);
   newSchema.annotations.obs.columns = schema.annotations.obs.columns.filter(
     v => v.name !== name
   );
-  return _reindex(newSchema);
+  return _reindexObsAnno(newSchema);
 }
 
 export function addObsAnnoColumn(schema, name, defn) {
-  const newSchema = _copy(schema);
+  const newSchema = _copyObsAnno(schema);
   newSchema.annotations.obs.columns.push(defn);
-  return _reindex(newSchema);
+  return _reindexObsAnno(newSchema);
 }
 
 export function removeObsAnnoCategory(schema, name, category) {
@@ -82,7 +93,7 @@ export function removeObsAnnoCategory(schema, name, category) {
   const idx = categories.indexOf(category);
   if (idx === -1) throw new Error("category does not exist");
 
-  const newSchema = _reindex(_copy(schema));
+  const newSchema = _reindexObsAnno(_copyObsAnno(schema));
 
   /* remove category.  Do not need to resort as this can't change presentation order */
   newSchema.annotations.obsByName[name].categories.splice(idx, 1);
@@ -98,7 +109,7 @@ export function addObsAnnoCategory(schema, name, category) {
   const idx = categories.indexOf(category);
   if (idx !== -1) throw new Error("category already exists");
 
-  const newSchema = _reindex(_copy(schema));
+  const newSchema = _reindexObsAnno(_copyObsAnno(schema));
 
   /* add category, retaining presentation sort order */
   const catAnno = newSchema.annotations.obsByName[name];
@@ -108,3 +119,17 @@ export function addObsAnnoCategory(schema, name, category) {
   ]);
   return newSchema;
 }
+
+export function addObsLayout(schema, layout) {
+  /* add or replace a layout */
+  const newSchema = _copyObsLayout(schema);
+  newSchema.layout.obs.push(layout);
+  return _reindexObsLayout(newSchema);
+}
+
+export function removeObsLayout(schema, name) {
+  /* remove a layout */
+  const newSchema = _copyObsLayout(schema);
+  newSchema.layout.obs = schema.layout.obs.filter(v => v.name !== name);
+  return _reindexObsLayout(newSchema);
+}

client/src/util/stateManager/universe.js

@@ -1,11 +1,9 @@
-import _ from "lodash";
-
 import { unassignedCategoryLabel } from "../../globals";
 import { decodeMatrixFBS } from "./matrix";
 import * as Dataframe from "../dataframe";
 import { isFpTypedArray } from "../typeHelpers";
-import { indexEntireSchema, sortAllCategorical } from "./schemaHelpers";
+import { indexEntireSchema } from "./schemaHelpers";
-import { isCategoricalAnnotation } from "./annotationsHelpers";
+import catLabelSort from "../catLabelSort";
 
 /*
 Private helper function - create and return a template Universe
@@ -74,9 +72,9 @@ function promoteTypedArray(o) {
   return new TyepdArrayCtor(o);
 }
 
-function AnnotationsFBSToDataframe(arrayBuffer) {
+export function matrixFBSToDataframe(arrayBuffers) {
   /*
-  Convert a Matrix FBS to a Dataframe.
+  Convert array of Matrix FBS to a Dataframe.
 
   The application has strong assumptions that all scalar data will be
   stored as a float32 or float64 (regardless of underlying data types).
@@ -86,77 +84,41 @@ function AnnotationsFBSToDataframe(arrayBuffer) {
   All float data from the server is left as is.  All non-float is promoted
   to an appropriate float.
   */
-  const fbs = decodeMatrixFBS(arrayBuffer, true); // leave in place
+  if (!Array.isArray(arrayBuffers)) {
-  const columns = fbs.columns.map(c => {
+    arrayBuffers = [arrayBuffers];
-    if (isFpTypedArray(c) || Array.isArray(c)) return c;
+  }
-    return promoteTypedArray(c);
+  if (arrayBuffers.length === 0) {
-  });
+    return Dataframe.Dataframe.empty();
-  const df = new Dataframe.Dataframe(
-    [fbs.nRows, fbs.nCols],
-    columns,
-    null,
-    new Dataframe.KeyIndex(fbs.colIdx)
-  );
-  return df;
-}
-
-function LayoutFBSToDataframe(arrayBuffer) {
-  const fbs = decodeMatrixFBS(arrayBuffer, true);
-  if (fbs.columns.length < 2 || !fbs.columns.every(isFpTypedArray)) {
-    // We have strong assumptions about the shape & type of layout data.
-    throw new Error("Unexpected layout data type returned from server");
   }
 
+  const fbs = arrayBuffers.map(ab => decodeMatrixFBS(ab, true)); // leave in place
+  /* check that all FBS have same row dimensionality */
+  const { nRows } = fbs[0];
+  fbs.forEach(b => {
+    if (b.nRows !== nRows)
+      throw new Error("FBS with inconsistent dimensionality");
+  });
+  const columns = fbs
+    .map(fb =>
+      fb.columns.map(c => {
+        if (isFpTypedArray(c) || Array.isArray(c)) return c;
+        return promoteTypedArray(c);
+      })
+    )
+    .flat();
+  const colIdx = fbs.map(b => b.colIdx).flat();
+  const nCols = columns.length;
+
   const df = new Dataframe.Dataframe(
-    [fbs.nRows, fbs.nCols],
+    [nRows, nCols],
-    fbs.columns,
+    columns,
     null,
-    new Dataframe.KeyIndex(fbs.colIdx)
+    new Dataframe.KeyIndex(colIdx)
   );
   return df;
 }
 
-function reconcileSchemaCategoriesWithSummary(universe) {
+export function createUniverseFromResponse(configResponse, schemaResponse) {
-  /*
-  where we treat types as (essentially) categorical metadata, update
-  the schema with data-derived categories (in addition to those in
-  the server declared schema).
-
-  For example, boolean defined fields in the schema do not contain
-  explicit declaration of categories (nor do string fields).  In these
-  cases, add a 'categories' field to the schema so it is accessible.
-
-  In addition, we have a client-side convention (UI) that all writable
-  annotations must have an 'unassigned' category, even if it is not currently
-  in use.
-  */
-
-  universe.schema.annotations.obs.columns.forEach(s => {
-    if (
-      s.type === "string" ||
-      s.type === "boolean" ||
-      s.type === "categorical"
-    ) {
-      const categories = _.union(
-        s.categories ?? [],
-        universe.obsAnnotations.col(s.name).summarize().categories ?? []
-      );
-      s.categories = categories;
-    }
-
-    if (s.writable && s.categories.indexOf(unassignedCategoryLabel) === -1) {
-      s.categories = s.categories.concat(unassignedCategoryLabel);
-    }
-  });
-}
-
-export function createUniverseFromResponse(
-  configResponse,
-  schemaResponse,
-  annotationsObsResponse,
-  annotationsVarResponse,
-  layoutFBSResponse
-) {
   /*
   build & return universe from a REST 0.2 /config, /schema and /annotations/obs response
   */
@@ -167,41 +129,80 @@ export function createUniverseFromResponse(
   universe.schema = schema;
   universe.nObs = schema.dataframe.nObs;
   universe.nVar = schema.dataframe.nVar;
+
   /* add defaults, as we can't assume back-end will fully populate schema */
   if (!schema.layout.var) schema.layout.var = [];
   if (!schema.layout.obs) schema.layout.obs = [];
-
-  /* annotations */
-  universe.obsAnnotations = AnnotationsFBSToDataframe(annotationsObsResponse);
-  universe.varAnnotations = AnnotationsFBSToDataframe(annotationsVarResponse);
-  /* layout */
-  universe.obsLayout = LayoutFBSToDataframe(layoutFBSResponse);
-
-  /* sanity checks */
-  if (
-    universe.nObs !== universe.obsLayout.length ||
-    universe.nObs !== universe.obsAnnotations.length ||
-    universe.nVar !== universe.varAnnotations.length
-  ) {
-    throw new Error("Universe dimensionality mismatch - failed to load");
-  }
-
-  reconcileSchemaCategoriesWithSummary(universe);
-  sortAllCategorical(universe.schema);
   indexEntireSchema(universe.schema);
+  normalizeEntireSchema(universe.schema);
 
-  /* sanity checks */
+  return universe;
-  if (
+}
-    schema.annotations.obs.columns.some(
+
-      s => s.writable && !isCategoricalAnnotation(schema, s.name)
+function normalizeSchemaCategory(colSchema, col = undefined) {
-    )
+  const { type, writable } = colSchema;
-  ) {
+  if (type === "string" || type === "boolean" || type === "categorical") {
+    let categories = [
+      ...new Set([
+        ...(colSchema.categories ?? []),
+        ...(col?.summarize?.().categories ?? [])
+      ])
+    ];
+    if (writable && categories.indexOf(unassignedCategoryLabel) === -1) {
+      categories = categories.concat(unassignedCategoryLabel);
+    }
+    colSchema.categories = categories;
+  } else if (writable) {
     throw new Error(
       "Writable continuous obs annotations are not supported - failed to load"
     );
   }
 
-  return universe;
+  if (colSchema.categories) {
+    colSchema.categories = catLabelSort(writable, colSchema.categories);
+  }
+}
+
+function normalizeEntireSchema(schema) {
+  // currently only needed for obsAnnotations
+  schema.annotations.obs.columns.forEach(colSchema =>
+    normalizeSchemaCategory(colSchema)
+  );
+}
+
+export function addObsAnnotations(universe, df) {
+  const obsAnnotations = universe.obsAnnotations.withColsFromAll(df);
+  if (universe.nObs !== obsAnnotations.length) {
+    throw new Error("Universe dimensionality mismatch - failed to load");
+  }
+
+  // for all of the new data, reconcile with schema and sort categories.
+  const dfs = Array.isArray(df) ? df : [df];
+  const keys = dfs.map(d => d.colIndex.keys()).flat();
+  const { schema } = universe;
+  keys.forEach(k => {
+    const colSchema = schema.annotations.obsByName[k];
+    const col = obsAnnotations.col(k);
+    normalizeSchemaCategory(colSchema, col);
+  });
+
+  return { obsAnnotations, schema };
+}
+
+export function addVarAnnotations(universe, df) {
+  const varAnnotations = universe.varAnnotations.withColsFromAll(df);
+  if (universe.nVar !== varAnnotations.length) {
+    throw new Error("Universe dimensionality mismatch - failed to load");
+  }
+  return { varAnnotations };
+}
+
+export function addObsLayout(universe, df) {
+  const obsLayout = universe.obsLayout.withColsFromAll(df);
+  if (universe.nObs !== obsLayout.length) {
+    throw new Error("Universe dimensionality mismatch - failed to load");
+  }
+  return { obsLayout };
 }
 
 export function convertDataFBStoObject(universe, arrayBuffer) {

client/src/util/stateManager/world.js

@@ -5,8 +5,8 @@ import { isContinuousAnnotation } from "./annotationsHelpers";
 
 /*
 
-World is a subset of universe.   Most code should use world, and should
+World is a subset of universe.  Most code should use world, and should
-(generally) not use Universe.   World contains any per-obs or per-var data
+(generally) not use Universe.  World contains any per-obs or per-var data
 that must be consistent acorss the app when we view/manipulate subsets
 of Universe.
 
@@ -113,7 +113,6 @@ function clipDataframe(
 
 /*
 Create World with contents eq entire universe.   Commonly used to initialize World.
-If clipQuantiles
 */
 export function createWorldFromEntireUniverse(universe) {
   const world = templateWorld();
@@ -253,28 +252,53 @@ function deduceDimensionType(attributes, fieldName) {
   return dimensionType;
 }
 
+function addObsDimension(crossfilter, world, anno) {
+  /*
+  add single dimension to the crosfilter
+  */
+  const { obsAnnotations } = world;
+  if (obsAnnotations.hasCol(anno.name)) {
+    const dimType = deduceDimensionType(anno, anno.name);
+    const colData = obsAnnotations.col(anno.name).asArray();
+    const name = obsAnnoDimensionName(anno.name);
+    if (dimType === "enum") {
+      return crossfilter.addDimension(name, "enum", colData);
+    }
+    if (dimType) {
+      return crossfilter.addDimension(name, "scalar", colData, dimType);
+    }
+  }
+  return crossfilter;
+}
+
+export function addObsDimensions(crossfilter, world) {
+  /*
+  Add to crossfilter any dimension present in world.obsAnnotations
+  but not yet in the crossfilter
+  */
+  const schema = world.schema.annotations.obsByName;
+  const dimsWeNeed = world.obsAnnotations.colIndex.keys();
+  crossfilter = dimsWeNeed.reduce((xfltr, name) => {
+    const dimName = obsAnnoDimensionName(name);
+    if (xfltr.hasDimension(dimName)) return xfltr;
+    return addObsDimension(xfltr, world, schema[name]);
+  }, crossfilter);
+  return crossfilter;
+}
+
 export function createObsDimensions(crossfilter, world, XYdimNames) {
   /*
   create and return a crossfilter with a dimension for every obs annotation
   for which we have a supported type, *except* for the index column, indicated
   by schema.annotations.obs.index.
   */
-  const { schema, obsLayout, obsAnnotations } = world;
+  const { schema, obsLayout } = world;
   const indexName = schema.annotations.obs.index;
   const annoList = schema.annotations.obs.columns.filter(
     anno => anno.name !== indexName
   );
   crossfilter = annoList.reduce((xfltr, anno) => {
-    const dimType = deduceDimensionType(anno, anno.name);
+    return addObsDimension(xfltr, world, anno);
-    const colData = obsAnnotations.col(anno.name).asArray();
-    const name = obsAnnoDimensionName(anno.name);
-    if (dimType === "enum") {
-      return xfltr.addDimension(name, "enum", colData);
-    }
-    if (dimType) {
-      return xfltr.addDimension(name, "scalar", colData, dimType);
-    }
-    return xfltr;
   }, crossfilter);
 
   return crossfilter.addDimension(

client/src/util/typedCrossfilter/bitArray.js

@@ -98,6 +98,7 @@ class BitArray {
       if (lowestZeroBit) {
         this.bitmask[col] |= lowestZeroBit;
         dim = 32 * col + BitArray.ctz(lowestZeroBit);
+        break;
       }
     }
     return dim;

common.mk

@@ -0,0 +1,33 @@
+PROJECT_ROOT := $(shell git rev-parse --show-toplevel)
+PATH := $(PATH):$(PROJECT_ROOT)/scripts
+
+SHELL := env PATH='$(PATH)' /bin/bash
+
+# get_or_else_dev_env_default
+# - If a variable is defined, return its value
+# - Else return the default value from environment.dev
+define get_or_else_dev_env_default
+$(if $($(1)),$($(1)),$(shell sed -n 's/$(1)=\(.*\)/\1/p' < $(PROJECT_ROOT)/environment.default))
+endef
+
+export CXG_SERVER_PORT := $(call get_or_else_dev_env_default,CXG_SERVER_PORT)
+export CXG_CLIENT_PORT := $(call get_or_else_dev_env_default,CXG_CLIENT_PORT)
+export JEST_ENV := $(call get_or_else_dev_env_default,JEST_ENV)
+
+export CXG_SERVER_PORT
+export CXG_CLIENT_PORT
+export JEST_ENV
+
+# copy the client assests to a location known to the server
+# $(1) is the source of the client assets
+# $(2) is the destination
+define copy_client_assets
+	mkdir -p $(2)/common/web/static/img
+	mkdir -p $(2)/common/web/static/js
+	mkdir -p $(2)/common/web/templates/
+	cp $(1)/index.html $(2)/common/web/templates/
+	cp -r $(1)/static $(2)/common/web/
+	cp $(1)/favicon.png $(2)/common/web/static/img
+	cp $(1)/service-worker.js $(2)/common/web/static/js/
+endef
+

dev_docs/developer_guidelines.md

@@ -9,6 +9,12 @@
 
 **All instructions are expected to be run from the top level cellxgene directory unless otherwise specified.**
 
+### Environment
+
+For all `make` commands, `common.mk` automatically checks whether required environment variables are set and, if they are not set, assigns them default values from `environment.default`.
+
+You can set these environment variables manually with the `export` shell command, as in `export JEST_ENV=debug`.
+
 ## Running test suite
 Client and server tests run on Travis CI for every push, PR, and commit to master on github. End to end tests run nightly on master only. 
 
@@ -18,21 +24,19 @@ Steps to run the all unit tests:
 1. `make dev-env`
 1. `make unit-test`
 
+To run unit tests for the `client` code only:
+1. Start in the project root directory
+1. `cd client`
+1. `make unit-test`
+
 ### End to end tests
 
-End to end tests use two env variables:
+To run E2E tests, run `cd client` and `make smoke-test`
-* `JEST_ENV` - environment to run end to end tests. Default `dev`
-    * `prod` - run headless with no slowdown, chromium will not open.
-    * `dev` - opens chromimum, runs tests with minimal slowdown, close on exit.
-    * `debug` - opens chromium, runs tests with 100ms slowdown, dev tools open, chrome stays open on exit.
-* `JEST_CXG_PORT` - port that end to end tests are being run on. Default `3000` (client hosted port).
 
-On CI the end to end tests are run with `JEST_ENV` set to `prod` using the `smoke-test` make target.
+The `JEST_ENV` environment variable enables the following E2E test options:
-
+* `dev` - opens chromimum, runs tests with minimal slowdown, close on exit.
-To run end to end tests as they will be run on CI use the following command:
+* `debug` - opens chromium, runs tests with 100ms slowdown, dev tools open, chrome stays open on exit.
-```
+* `prod` - run headless with no slowdown, chromium will not open.
-JEST_ENV=prod JEST_CXG_PORT=5000 make pydist install-dist dev-env smoke-test
-```
 
 Run end to end tests interactively during development
 1. cellxgene should be installed as [specified in client dev](#install-1)
@@ -40,6 +44,11 @@ Run end to end tests interactively during development
 1. Run `npm run e2e` or `make e2e` from the `client` directory
 1. To debug a failing test `export JEST_ENV='debug'` and re-run.
 
+To run end to end tests _exactly_ as they will be run on CI use the following command:
+```
+JEST_ENV=prod make pydist install-dist dev-env smoke-test
+```
+
 ## Server dev
 ### Install
 * Build the client and put static files in place: `make build-for-server-dev`

dev_docs/developer_scripts.md

@@ -1,56 +1,8 @@
-## Node scripts in package.json
+# Developer convenience methods
 
-#### backend-dev
+## Makefile
 
-**About** This scripts enables FE developers to run the rest API necessary to back the development server for the front end. It is intended to ensure that the FE developer gets the current version of the backend with a single command and no knowledge of python necessary. It creates and activates a virtual environment and installs cellxgene from the current branch.
+Documentation for the `Makefile` targets in the project root directory.
-
-**Requires** Python3.6 - `virtual-env`, `pip`
-
-**Usage** `npm run backend-dev <dataset>` then you can launch the node development server to serve the current state of the FE `npm run start`
-
-**Tips** Developers will probably want to run this in parallel with the node dev server. You can either do this by running each in a separate termanial window or by running `backend-dev` in the background (add an `&` at the end of the command to run in the background: `npm run backend-dev <dataset> &`).
-
-**Breakdown**
-
-| command                                  | purpose                                                   |
-| ---------------------------------------- | --------------------------------------------------------- |
-| python3.6 -m venv cellxgene              | creates cellxgene virtual environment                     |
-| source cellxgene/bin/activate            | activates virtual environment                             |
-| yes \| pip uninstall cellxgene \|\| true | uninstalls cellxgene (if installed)                       |
-| pip install -e ..                        | installs current local version of cellxgene               |
-| cellxgene launch                         | launches cellxgene (must supply dataset as last parameter |
-
-### Test scripts
-
-#### test
-
-**About** Run test locally. In order for this command to succeed you will need to give it a specific unit test to run. It won't pass if you run all tests as may be expected. This is because the unit tests and end to end (e2e) tests require different testing environments.
-
-**Usage** `npm run test <test file or pattern>`
-
-#### unit-test
-
-**About** Runs all unit tests. It excludes any tests in the e2e folder. This is used by travis to run unit tests.
-
-**Usage** `npm run unit-test`
-
-#### smoke-test
-
-**About** Starts backend development server and runs end to end tests. This is what travis runs. It depends on the `e2e` and the `start-server-for-test` node scripts. One starts the server, the other runs the tests. If developing a front-end feature and just checking if tests pass, this is probabaly the one you want to run.
-
-**Requirements** Must set env variable for port. In terminal `export JEST_ENV='dev'` `export JEST_CXG_PORT=5000`
-
-**Usage** `npm run smoke-test`
-
-#### e2e
-
-**About** Runs backend tests without starting the server. You will need to start the rest api separately with the pbmc3k.h5ad file.
-
-**Requirements** Must set env variable for port. In terminal `export JEST_ENV='dev'` `export JEST_CXG_PORT=5000`
-
-**Usage** `npm run e2e`
-
-## makefile
 
 ### Build commands
 
@@ -91,7 +43,6 @@ Installs requirements files
 
 ```
 dev-env - installs requirements and requirments-dev (for building code)
-gui-env - installs requirements and requirments-dev and requirments-gui (for building native app)
 ```
 
 ### install commands
@@ -107,14 +58,67 @@ install-dist - installs from local dist folder
 uninstall - uninstalls cellxgene
 ```
 
-### gui
+## Client Makefile
 
-Commands for building the native app
+The following phony `make` targets in `client/Makefile` are convenience methods for getting you up and developing.
 
-```
+### Runner scripts
-build-assets - builds the image and icon assets for the gui to pull from
+
-gui-spec-osx - creates the initial spec file for osx, do not run unless you are starting from scratch, one time only
+#### start-frontend
-gui-spec-windows - creates the initial spec file for windows, do not run unless you are starting from scratch, one time only
+
-gui-build-osx - builds the app from the osx spec file
+**About** Serve the current client javascript independently from the `server` code.
-gui-build-windows - builds the app from the windows spec file
+
-```
+**Requires**
+* The server to be running. Best way to do this is with `make backend-dev`
+* `make ci` to install the necessary node modules
+
+**Usage** `make start-frontend`
+
+#### backend-dev
+
+**About** This script enables FE developers to run the REST API necessary to back the development server for the front end. It is intended to ensure that the FE developer gets the current version of the backend with a single command and no knowledge of python necessary. It creates and activates a virtual environment and installs cellxgene from the current branch.
+
+**Requires** Python3.6 - `virtual-env`, `pip`
+
+**Usage** `make backend-dev`. Optionally, you can then launch the node development server to serve the current state of
+the FE with `make start-frontend`. You can also select a specific dataset using `DATASET=<dataset path> make backend-dev`.
+You can also use `CXG_OPTIONS` to pass options to the `cellxgene launch` command, as in
+`CXG_OPTIONS='--experimental-annotations --experimental-annotations-file annotations.csv' make backend-dev`.
+
+**Tips** Developers will probably want to run this in parallel with the node dev server. You can either do this by running each in a separate termanial window or by running `backend-dev` in the background (add an `&` at the end of the command to run in the background: `DATASET=<dataset> make backend-dev &`).
+
+**Breakdown**
+
+| command                                  | purpose                                                   |
+| ---------------------------------------- | --------------------------------------------------------- |
+| python3.6 -m venv cellxgene              | creates cellxgene virtual environment                     |
+| source cellxgene/bin/activate            | activates virtual environment                             |
+| yes \| pip uninstall cellxgene \|\| true | uninstalls cellxgene (if installed)                       |
+| pip install -e .                        | installs current local version of cellxgene               |
+| cellxgene launch                         | launches cellxgene (must supply dataset as last parameter) |
+
+### Test scripts
+
+#### test
+
+**About** Run test locally. In order for this command to succeed you will need to give it a specific unit test to run. It won't pass if you run all tests as may be expected. This is because the unit tests and end to end (e2e) tests require different testing environments.
+
+**Usage** `make test`
+
+#### unit-test
+
+**About** Runs all unit tests. It excludes any tests in the e2e folder. This is used by travis to run unit tests.
+
+**Usage** `make unit-test`
+
+#### smoke-test
+
+**About** Starts backend development server and runs end to end tests. This is what travis runs. It depends on the `e2e` and the `backend-dev` targets. One starts the server, the other runs the tests. If developing a front-end feature and just checking if tests pass, this is probabaly the one you want to run.
+
+**Usage** `make smoke-test`
+
+#### e2e
+
+**About** Runs backend tests without starting the server. You will need to start the rest api separately with the pbmc3k.h5ad file. Note you can use the `JEST_ENV` environment variable to change how JEST runs in the browser.
+
+**Usage** `make e2e`

docs/posts/annotations.md

@@ -13,7 +13,7 @@ We are _piloting_ a new feature in cellxgene that enables users to create and ed
 
 To get started, run:
 ```
-cellxgene launch mydata.h5ad --experimental-annotations
+cellxgene launch mydata.h5ad
 ```
 
 To preserve data provenance, **`cellxgene` does not alter the input h5ad file**.  Rather, newly-created annotations are saved in a specified CSV file:  
@@ -21,8 +21,7 @@ To preserve data provenance, **`cellxgene` does not alter the input h5ad file**.
 annotations-directory/name-########.csv
 ```
 
-- The default `annotations-directory` is  your current working directory (i.e., the directory you were in when you started cellxgene).
+- The default `annotations-directory` is your current working directory (i.e., the directory you were in when you started cellxgene).
-
 - You will be prompted to enter a name for your annotations the first time you create a new category.  
 - We also assign a unique identifier in the form of an 8-character suffix, `########`; this helps cellxgene identify your file to avoid overwriting your work.
 
@@ -36,10 +35,7 @@ There are two options for updating draft annotations.
 
 ### Autodetect annotations csv
 
-Cellxgene can automatically find and reload your draft annotations in editable mode like so:  
+Cellxgene will automatically find and reload your draft annotations in editable mode.
-```
-annotations-directory$ cellxgene launch mydata.h5ad --experimental-annotations
-```
 
 This assumes that:
 1 - The h5ad filename is the same  
@@ -52,7 +48,7 @@ This assumes that:
 
 If you'd like to specify the complete file path for your annotations, you can do so by running:  
 ```
-cellxgene launch mydata.h5ad --experimental-annotations --experimental-annotations-file path/to/myfile.csv
+cellxgene launch mydata.h5ad --annotations-file path/to/myfile.csv
 ```
 
 Any changes you make will be reflected in the original CSV. If the file does not exist, it will be created.  
@@ -83,7 +79,7 @@ Specifying a single file name for multiple contributors will result in data over
 
 To specify an output directory, run:  
 ```
-cellxgene launch mydata.h5ad --experimental-annotations --experimental-annotations-output-dir path/to/annotations-directory/
+cellxgene launch mydata.h5ad --annotations-dir path/to/annotations-directory/
 ```
 
 For each user, annotations will be saved as follows:  
@@ -93,8 +89,6 @@ For each user, annotations will be saved as follows:
 
 
 
-
-
 ## FAQ
 
 ### How do I know my annotations are saved?
@@ -113,4 +107,4 @@ This is most likely because the h5ad file you are working with is not the origin
 We place a small cookie (file) in your browser that identifies where your draft annotations are saved. This file never leaves your machine, and is never sent to the cellxgene team or anyone else.
 
 ### I have feedback and ideas for you!
-Wonderful! This is a very new and complex feature; we would _love_ to [hear your feedback](contact) :)
+Wonderful! This is a relatively new feature; we would _love_ to [hear your feedback](contact) :)

docs/posts/contribute.md

@@ -1,4 +1,5 @@
 # Code of conduct
+
 We warmly welcome contributions from the community!
 
 To ensure a welcoming experience for our entire community, this project adheres to the Contributor Covenant
@@ -13,12 +14,16 @@ If you have any questions about any of this stuff, just ask! :)
 We'd love to hear from you! Please submit any bug reports and feature requests through [Github issues](https://github.com/chanzuckerberg/cellxgene/issues).
 
 # Direct contributions
+
 ## Getting started
 
-If you are interested in working on `cellxgene` development, you'll need to use git to make a copy of the [project repository](https://www.youtube.com/watch?v=A-4WltCTVms&list=PLe6EXFvnTV7-_41SpakZoTIYCgX4aMTdU&index=2&t=0s) and share your changes.
+If you are interested in working on `cellxgene` development, you'll need to use git to make a copy of the [project repository](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/working-with-forks) and share your changes.
-If you're new to git, we recommend [GitKraken](https://www.gitkraken.com/) for an intuitive interface.
 
-Please submit any direct contributions by [forking the repository](https://www.youtube.com/watch?v=Lb4yvfrX_7I&list=PLe6EXFvnTV7-_41SpakZoTIYCgX4aMTdU&index=3&t=9s), creating a branch, and [submitting a Pull Request](https://www.youtube.com/watch?v=2VX1ISk9XH8&list=PLe6EXFvnTV7-_41SpakZoTIYCgX4aMTdU&index=9&t=0s).
+We have several "rules" (strong suggestions really) for contributions:
+
+1. If your contribution is complex, adds new features, new UI design or otherwise warrants discussion, we highly recommend that you submit a github issue, and engage other contributors in a discussion about the details of your proposed PR. This will save you time in the long run, as many details and decisions can be hashed out ahead-of-time.
+
+2. Please submit any direct contributions by [forking the repository](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/working-with-forks), creating a feature branch, and [submitting a Pull Request](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request).
 
 First, you'll need the following installed on your machine
 
@@ -33,13 +38,13 @@ git clone https://github.com/chanzuckerberg/cellxgene.git
 
 This is enough to get you started with editing documentation. If you'd like to contribute code:
 
-Build the client web assets by calling `make` from inside the `cellxgene` folder
+Build the client web assets from inside the `cellxgene` folder
 
 ```
-make
+make build-for-server-dev
 ```
 
-Install all requirements (we recommend doing this inside a [virtual environment](install))
+Install all requirements (we recommend doing this inside a [virtual environment](https://docs.python.org/3/tutorial/venv.html))
 
 ```
 pip install -e .

docs/posts/hosted.md

@@ -23,9 +23,9 @@ The following configuration options require special consideration in any multi-u
 
 `--disable-diffexp`: the differential expression computation can be resource intensive, in particular for large datasets. If many differential expression calculation requests are made in rapid sequence, it may cause the server CPU or memory resources to be exhausted, and impact the ability of other users to access data. This command line option will disable the differential expression feature, including the removal of the `Differential expression` button.
 
-`--experimental-annotations`: this feature, which is disabled by default, may not be appropriate for hosted environments. It will write to the local file system, and in extreme cases could be used to abuse (or exceed) file system capacity on the hosting server.
+`--disable-annotations`: annotations, which is enabled by default, may not be appropriate for hosted environments. It will write to the local file system, and in extreme cases could be used to abuse (or exceed) file system capacity on the hosting server. We recommend disabling this with this flag.
 
-`--experimental-annotations-file`: this specifies a single file for all end-user annotations, and is incompatible with hosted or multi-user use of cellxgene. Using it will cause loss of user annotation data (ie, the CSV file will be overwritten). If you wish to explore using the _experimental_ annotations feature in a multi-user environment, please refer to the [annotations documentation](annotations).
+`--annotations-file`: this specifies a single file for all end-user annotations, and is incompatible with hosted or multi-user use of cellxgene. Using it will cause loss of user annotation data (ie, the CSV file will be overwritten). If you wish to explore using the annotations feature in a multi-user environment, please refer to the [annotations documentation](annotations).
 
 ## Community software projects
 
@@ -42,7 +42,7 @@ If you know of other solutions, drop us a note and we'll add to this list.
 
 Clicking on the following button will forward you to Heroku to begin the deployment process:
 
-<a href="https://heroku.com/deploy?template=https://github.com/chanzuckerberg/cellxgene/tree/heroku">
+<a href="https://heroku.com/deploy?template=https://github.com/chanzuckerberg/cellxgene">
  <img src="https://www.herokucdn.com/deploy/button.svg" alt="Deploy">
 </a>
 

docs/posts/launch.md

@@ -46,7 +46,7 @@ cellxgene launch s3://mybucket.s3-us-west-2.amazonaws.com/mydata.h5ad
 
 `--open` automatically opens the web browser after launching (caveat: only works on some operating systems).
 
-`--experimental-annotations`, `--experimental-annotations-file` & `--experimental-annotations-output-dir` all have to do with an experimental feature to allow users to create new categorical annotations in the application. We have a [whole separate page](annotations) about their usage! :)
+`--disable-annotations`, `--annotations-file` & `--annotations-dir` all have to do with creating new categorical annotations in the application. We have a [whole separate page](annotations) about their usage! :)
 
 `--diffexp-lfc-cutoff` as explained [in the methods](methods), genes are only returned in differential expression if the effect size is above the specified threshold for log fold change. Defaults to 0.01.
 

environment.default

@@ -0,0 +1,4 @@
+# Default environment variables for cellxgene development
+CXG_SERVER_PORT=5005
+CXG_CLIENT_PORT=3000
+JEST_ENV=dev

experiments/heroku/README.md

@@ -4,7 +4,7 @@
 
 Clicking on the following button will forward you to Heroku to begin the deployment process:
 
-<a href="https://heroku.com/deploy?template=https://github.com/chanzuckerberg/cellxgene/tree/heroku">
+<a href="https://heroku.com/deploy?template=https://github.com/chanzuckerberg/cellxgene">
  <img src="https://www.herokucdn.com/deploy/button.svg" alt="Deploy">
 </a>
 

heroku.yml

@@ -2,4 +2,4 @@ build:
   docker:
     web: experiments/heroku/Dockerfile
 run:
-  web: curl -o dataset.h5ad $DATASET && cellxgene launch dataset.h5ad --host 0.0.0.0 --port $PORT
+  web: cellxgene launch $DATASET --host 0.0.0.0 --port $PORT

pyproject.toml

@@ -17,9 +17,8 @@ exclude = '''
     | buck-out
     | build
     | dist
-    | server/app/util/fbs/NetEncoding
+    | server/data_common/fbs/NetEncoding
   )/
-  | server/gui/cellxgene_rc.py
 
 )
 '''

scripts/await_port

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+import argparse
+import errno
+import socket
+import sys
+import time
+
+# await_port
+#
+# Functionality:
+# 1. Wait for a process to be listening on a port - await_port will poll the address with a bind request and close it
+#    until another process takes the address.
+# 2. Wait for a port to be free - await_port will poll the address with a bind request and close it until the port is
+#    free.
+
+parser = argparse.ArgumentParser(description='Wait for a port either to be bound to or to be free.')
+parser.add_argument('port', type=int, help='port to wait for')
+parser.add_argument('--sleep-increment', type=int, default=2.0, help='sleep duration in between checks of the port')
+parser.add_argument('--timeout', type=int, default=60.0, help='How long should we wait before timing out?')
+parser.add_argument('--await-free', default=False, action='store_true',
+                    help='Wait for the port to be active or free?')
+
+args = parser.parse_args()
+
+start = time.time()
+
+port_in_use = None
+
+desired_state = 'free' if args.await_free else 'in use'
+
+
+def condition_reached(port_in_use):
+    return not port_in_use if args.await_free else port_in_use
+
+
+while True:
+    # halt if we've reached the timeout
+    elapsed = time.time() - start
+    if elapsed > args.timeout:
+        print(f"Timed out while waiting for port {args.port}")
+        sys.exit(1)
+
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        try:
+            s.bind(('localhost', args.port))
+            port_in_use = False
+        except socket.error as e:
+            if e.errno != errno.EADDRINUSE:
+                raise e
+            port_in_use = True
+
+    if condition_reached(port_in_use):
+        break
+
+    print(f"Waiting for port {args.port} to be {desired_state}... sleeping for {args.sleep_increment} seconds.")
+    time.sleep(args.sleep_increment)
+
+print(f"Port {args.port} is {desired_state}!")

scripts/start_server_and_test

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+set -e
+
+# start_server_and_test
+# Replicates the basic functionality of the start-server-and-test npm package
+# https://www.npmjs.com/package/start-server-and-test
+
+if [[ $# -ne 3 ]]; then
+  echo "Usage: start_server_and_test \'[start server script]\' [port] \'[test script]\'"
+  exit 1
+fi
+
+START_SERVER_SCRIPT="$1"
+PORT="$2"
+TEST_SCRIPT="$3"
+
+await_port --await-free "$PORT"
+
+eval "$START_SERVER_SCRIPT &"
+SERVER_PID=$!
+
+function finish {
+  kill $SERVER_PID || true
+}
+
+trap finish EXIT
+
+await_port "$PORT"
+
+eval "$TEST_SCRIPT"

server/Makefile

@@ -1,8 +1,17 @@
+include ../common.mk
+
 .PHONY: clean
 clean:
-	rm -f app/web/templates/index.html
+	rm -f common/web/templates/index.html
-	rm -rf app/web/static
+	rm -rf common/web/static
 
 .PHONY: unit-test
 unit-test:
-	pytest -s test
+	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
+		--source=app,cli,common,compute,converters,data_anndata,data_common,data_cxg \
+		--omit=.coverage,data_common/fbs/NetEncoding,venv \
+		-m unittest discover \
+		--start-directory test/ \
+		--top-level-directory ../ \
+		--verbose
+

server/__init__.py

@@ -1 +1 @@
-__version__ = "0.14.1"
+__version__ = "0.15.0"