Bump version: 0.16.6-rc.0 → 0.16.6

* Updates due dependency version changes.

h5py recently changes and now values once returned as str are now returned as bytes.
This would have caused a much larger change, so instead the version is restricted to <3.0.0.

This caused the bulk of the testing failues.
A few other changes were needed to make a few other tests pass.

 #1959

.bumpversion.cfg

@@ -1,5 +1,16 @@
 [bumpversion]
-current_version = 0.14.0
+current_version = 0.16.6
+commit = True
+parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:-(?P<prerel>rc)\.(?P<prerelversion>\d+))?
+serialize = 
+	{major}.{minor}.{patch}-{prerel}.{prerelversion}
+	{major}.{minor}.{patch}
+
+[bumpversion:part:prerel]
+optional_value = release
+values = 
+	rc
+	release
 
 [bumpversion:file:setup.py]
 search = version="{current_version}"
@@ -12,4 +23,3 @@ replace = "version": "{new_version}"
 [bumpversion:file:server/__init__.py]
 search = __version__ = "{current_version}"
 replace = __version__ = "{new_version}"
-

.codecov.yml

@@ -0,0 +1,8 @@
+coverage:
+  status:
+    project:
+      default:
+        informational: true
+    patch:
+      default:
+        informational: true

.github/ISSUE_TEMPLATE/---bug-report.md

@@ -0,0 +1,32 @@
+---
+name: "\U0001F41E Bug report"
+about: Create a report to identify a bug in cellxgene
+title: "[BUG]"
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Version (please complete the following information):**
+ - Desktop or hosted?:
+ - Browser (if hosted) [e.g. chrome, safari]:
+ - Version [e.g. 0.13.0]:
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/---feature-request.md

@@ -0,0 +1,20 @@
+---
+name: "\U0001F4A1 Feature request"
+about: Suggest an idea for this project
+title: "[FEATURE REQUEST]"
+labels: user request
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/---question-clarification.md

@@ -0,0 +1,10 @@
+---
+name: "\U0001F9D0 Question/Clarification"
+about: Ask a question or for a clarification
+title: "[QUESTION]"
+labels: question
+assignees: ''
+
+---
+
+

.github/workflows/compatibility_tests.yml

@@ -0,0 +1,108 @@
+name: Compatibility Tests
+
+on:
+  schedule:
+    - cron: '0 8 7 * 2'
+  push:
+    branches:
+      - main
+
+env:
+  JEST_ENV: prod
+
+jobs:
+  docker-build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Build docker image
+        run: docker build .
+
+  cellxgene-main-with-python-and-anndata-versions:
+    name: python versions x anndata versions
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8]
+        anndata-version: [0.6.22.post1, 0.7.1]
+        test-suite: [smoke-test, smoke-test-annotations]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        # 1. only install the dev requirements on top of what is in the cellxgene pip package
+        sudo apt-get update && sudo apt-get install -y libhdf5-serial-dev
+        sed -i 's/-r requirements.txt//' server/requirements-dev.txt
+        pip install -r server/requirements-dev.txt
+        # 2. install cellxgene
+        make pydist install-dist
+        # 3. install anndata
+        pip install anndata==${{ matrix.anndata-version }}
+        # workaround for anndata 0.6.22.post1 bug
+        [[ "0.6.22.post1" = "${{ matrix.anndata-version }}" ]] && pip install h5py==2.9.0 || true
+    - name: Tests
+      run: make unit-test ${{ matrix.test-suite }}
+
+  cellxgene-release-with-anndata-master:
+    name: cellxgene release with anndata master
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        test-suite: [smoke-test, smoke-test-annotations]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          path: cellxgene
+      - name: Install dependencies
+        run: |
+          cd cellxgene
+          # 1. only install the dev requirements on top of what is in the cellxgene pip package
+          make dev-env-client
+          sed -i 's/-r requirements.txt//' server/requirements-dev.txt
+          pip install -r server/requirements-dev.txt
+          # 2. install cellxgene
+          pip install --upgrade cellxgene
+          # 3. install anndata
+          pip install git+https://github.com/theislab/anndata
+      - name: Tests
+        run: cd cellxgene && make unit-test ${{ matrix.test-suite }}
+
+  cellxgene-main-with-anndata-master:
+    name: cellxgene main with anndata master
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        test-suite: [smoke-test, smoke-test-annotations]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          path: cellxgene
+      - name: Install dependencies
+        run: |
+          cd cellxgene
+          sed -i -E 's/^anndata[>=]=[0-9]+.[0-9]+.[0-9]+$/anndata/g' server/requirements.txt
+          make pydist install-dist dev-env
+          pip install git+https://github.com/theislab/anndata
+      - name: Tests
+        run: cd cellxgene && make unit-test ${{ matrix.test-suite }}

.github/workflows/deploy.yml

@@ -0,0 +1,13 @@
+name: Deploy via single cell infra repo
+
+on:
+  push:
+    branches: main
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: repository dispatch
+        run: |
+          curl -XPOST -u czi-sci-single-cell-eng:${{secrets.SCI_GITHUB_TOKEN}} -H "Accept: application/vnd.github.everest-preview+json"  -H "Content-Type: application/json" https://api.github.com/repos/chanzuckerberg/single-cell-infra/dispatches --data '{"event_type": "cellxgene-hook"}'

.github/workflows/push_tests.yml

@@ -0,0 +1,133 @@
+name: Push Tests
+
+on:
+  push:
+    branches: main
+  pull_request:
+    branches: "*"
+
+env:
+  JEST_ENV: prod
+  CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: |
+          git fetch --depth=1 origin +${{github.base_ref}}
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Node cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: |
+          pip install flake8
+          pip install black
+          cd client
+          npm install
+      - name: Format with black and lint with flake8
+        run: |
+          make lint-server
+      - name: Lint src with eslint
+        working-directory: ./client
+        run: |
+          make lint
+
+  unit-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Python cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Node cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: make pydist install-dist dev-env-server
+      - name: Unit tests
+        run: |
+          make unit-test
+          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF backend,python,unitTest
+          cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
+
+  smoke-tests:
+    runs-on: macos-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Python cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Node cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: make pydist install-dist
+      - name: Smoke tests (without annotations feature)
+        run: |
+          cd client && make smoke-test
+          ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTest
+
+  smoke-tests-annotations:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Python cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Node cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: make pydist install-dist
+      - name: Smoke tests (with annotations feature)
+        run: |
+          cd client && make smoke-test-annotations
+          ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTestAnnotations

.gitignore

@@ -3,6 +3,8 @@ node_modules/
 
 # coverage
 coverage
+coverage.xml
+.coverage
 .nyc_output
 
 .pytest_cache/
@@ -17,12 +19,14 @@ venv/
 cellxgene/
 
 # client build
-server/app/web/static/css/
-server/app/web/static/img/
-server/app/web/static/media/
-server/app/web/static/fonts/
-server/app/web/static/js/
-server/app/web/templates/index\.html
+server/common/web/static/*
+server/common/web/templates/
+server/common/web/csp-hashes.json
+
+# eb build
+artifact.dir
+artifact.zip
+customize
 
 # Jupyter Notebook
 .ipynb_checkpoints
@@ -39,7 +43,14 @@ npm-debug.log
 __pycache__
 *.DS_Store*
 data
+tags
+myconfig.yaml
 
 # Jekyll
 docs/_site/
 docs/Gemfile.lock
+
+client/.eslintcache
+
+# E2E Testing
+ignoreE2E*

.travis.yml

@@ -1,28 +0,0 @@
-language: python
-dist: xenial
-sudo: required
-node_js:
-  - 10
-cache:
-  - pip
-  - npm
-install:
-  - set -eo pipefail
-  - pip install flake8
-  - make pydist install-dist dev-env
-
-jobs:
-  include:
-    - name: "Branch Tests 3.7"
-      python: "3.7"
-      script: make build-client lint unit-test
-    - name: "Branch Tests 3.6"
-      python: "3.6"
-      script: make build-client lint unit-test
-    - name: "Docker Build"
-      install: skip
-      python: "3.6"
-      script: docker build .
-    - name: "Smoke Tests"
-      python: "3.6"
-      script: make smoke-test

Dockerfile

@@ -4,7 +4,8 @@ ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
 
 RUN apt-get update && \
-    apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests && \
+    apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests python3-aiohttp && \
+    python3 -m pip install --upgrade pip && \
     pip3 install cellxgene
 
 ENTRYPOINT ["cellxgene"]

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2013
+Copyright (c) 2017-2020 Chan Zuckerberg Initiative
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in
@@ -17,4 +17,4 @@ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
 FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

MANIFEST.in

@@ -1,5 +1,5 @@
-recursive-include server/app/web/templates *
-recursive-include server/app/web/static *
+recursive-include server/common/web/templates *
+recursive-include server/common/web/static *
 
 include server/requirements.txt
 include server/requirements-prepare.txt

Makefile

@@ -1,3 +1,5 @@
+include common.mk
+
 BUILDDIR := build
 CLIENTBUILD := $(BUILDDIR)/client
 SERVERBUILD := $(BUILDDIR)/server
@@ -5,7 +7,6 @@ CLEANFILES :=  $(BUILDDIR)/ client/build build dist cellxgene.egg-info
 
 PART ?= patch
 
-
 # CLEANING
 .PHONY: clean
 clean: clean-lite clean-server clean-client
@@ -21,37 +22,25 @@ clean-%:
 
 # BUILDING PACKAGE
 
-.PHONY: build
-build: clean build-cli
-	@echo "done"
-
 .PHONY: build-client
 build-client:
 	cd client && $(MAKE) ci build
 
-.PHONY: build-cli
-build-cli: build-client
-	git ls-files server/ | cpio -pdm $(BUILDDIR)
+.PHONY: build
+build: clean build-client
+	git ls-files server/ | grep -v 'server/test/' | cpio -pdm $(BUILDDIR)
 	cp -r client/build/  $(CLIENTBUILD)
-	mkdir -p $(SERVERBUILD)/app/web/static/img
-	mkdir -p $(SERVERBUILD)/app/web/templates/
-	cp $(CLIENTBUILD)/index.html $(SERVERBUILD)/app/web/templates/
-	cp -r $(CLIENTBUILD)/static $(SERVERBUILD)/app/web/
-	cp $(CLIENTBUILD)/favicon.png $(SERVERBUILD)/app/web/static/img
-	cp $(CLIENTBUILD)/service-worker.js $(SERVERBUILD)/app/web/static/js/
+	$(call copy_client_assets,$(CLIENTBUILD),$(SERVERBUILD))
 	cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR)
 
 # If you are actively developing in the server folder use this, dirties the source tree
 .PHONY: build-for-server-dev
 build-for-server-dev: clean-server build-client
-	mkdir -p server/app/web/static/img
-	mkdir -p server/app/web/static/js
-	mkdir -p server/app/web/templates/
-	cp client/build/index.html server/app/web/templates/
-	cp -r client/build/static server/app/web/
-	cp client/build/favicon.png server/app/web/static/img
-	cp client/build/service-worker.js server/app/web/static/js/
+	$(call copy_client_assets,client/build,server)
 
+.PHONY: copy-client-assets
+copy-client-assets:
+	$(call copy_client_assets,client/build,server)
 
 # TESTING
 .PHONY: test
@@ -67,21 +56,39 @@ unit-test-%:
 smoke-test:
 	cd client && $(MAKE) smoke-test
 
+.PHONY: smoke-test-annotations
+smoke-test-annotations:
+	cd client && $(MAKE) smoke-test-annotations
+
+.PHONY: test-db
+test-db:
+	cd server && $(MAKE) test-db
+
+
 # FORMATTING CODE
 
 .PHOHY: fmt
 fmt: fmt-client fmt-py
 
+.PHONY: fmt-client
 fmt-client:
 	cd client && $(MAKE) fmt
 
+.PHONY: fmt
 fmt-py:
 	black .
 
 .PHONY: lint
-lint:
-	flake8 server
+lint: lint-server lint-client
 
+.PHONY: lint-server
+lint-server: fmt-py
+	flake8 server --per-file-ignores='server/test/fixtures/dataset_config_outline.py:F821 server/test/fixtures/server_config_outline.py:F821 server/test/performance/scale_test_annotations.py:E501'
+
+
+.PHONY: lint-client
+lint-client:
+	cd client && $(MAKE) lint
 
 # CREATING DISTRIBUTION RELEASE
 
@@ -93,22 +100,32 @@ pydist: build
 
 # RELEASE HELPERS
 
-# create new version to commit to master
-.PHONY: release-stage-1
-release-stage-1: dev-env bump clean-lite gen-package-lock
+# Create new version to commit to main
+.PHONY: create-release-candidate
+create-release-candidate: dev-env bump-version clean-lite gen-package-lock
 	@echo "Version bumped part:$(PART) and client built. Ready to commit and push"
 
-# build dist and release to dev pypi
-.PHONY: release-stage-2
-release-stage-2: dev-env pydist twine
+# Bump the release candidate version if needed (i.e. the previous release candidate had errors).
+.PHONY: recreate-release-candidate
+recreate-release-candidate: dev-env bump-release-candidate clean-lite gen-package-lock
+	@echo "Version bumped part:$(PART) and client built. Ready to commit and push"
+
+# Build dist and release to Test PyPI
+.PHONY: release-candidate-to-test-pypi
+release-candidate-to-test-pypi: dev-env pydist twine
 	@echo "Dist built and uploaded to test.pypi.org"
 	@echo "Test the install:"
 	@echo "    make install-release-test"
-	@echo "Then upload to Pypi prod:"
-	@echo "    make twine-prod"
 
-.PHONY: release-stage-final
-release-stage-final: twine-prod
+# Build final dist (gets rid of the rc tag) and release final candidate to TestPyPI
+.PHONY: release-final-to-test-pypi
+release-final-to-test-pypi: dev-env bump-release clean-lite gen-package-lock pydist twine
+	@echo "Final release dist built and uploaded to test.pypi.org"
+	@echo "Test the install:"
+	@echo "    make install-release-test"
+
+.PHONY: release-final
+release-final: twine-prod
 	@echo "Release uploaded to pypi.org"
 
 # DANGER: releases directly to prod
@@ -120,19 +137,32 @@ release-directly-to-prod: dev-env pydist twine-prod
 	@echo "    make install-release"
 
 .PHONY: dev-env
-dev-env:
+dev-env: dev-env-client dev-env-server
+
+.PHONY: dev-env-client
+dev-env-client:
 	cd client && $(MAKE) ci
+
+.PHONY: dev-env-server
+dev-env-server:
 	pip install -r server/requirements-dev.txt
 
-.PHONY: gui-env
-gui-env: dev-env
-	pip install -r server/requirements-gui.txt
-
-# give PART=[major, minor, part] as param to make bump
-.PHONY: bump
-bump:
+# Set PART=[major, minor, patch] as param to make bump.
+# This will create a release candidate. (i.e. 0.16.1 -> 0.16.2-rc.0 for a patch bump)
+.PHONY: bump-version
+bump-version:
 	bumpversion --config-file .bumpversion.cfg $(PART)
 
+# Increments the release candidate version (i.e. 0.16.2-rc.1 -> 0.16.2-rc.2)
+.PHONY: bump-release-candidate
+bump-release-candidate:
+	bumpversion --config-file .bumpversion.cfg prerelversion --allow-dirty
+
+# Finalizes the release candidate by removing the release candidate tag (i.e. 0.16.2-rc.2 -> 0.16.2).
+.PHONY: bump-release
+bump-release:
+	bumpversion --config-file .bumpversion.cfg prerel --allow-dirty
+
 .PHONY: twine
 twine:
 	twine upload --repository-url https://test.pypi.org/legacy/ dist/*
@@ -149,13 +179,6 @@ gen-package-lock:
 
 # INSTALL
 
-# setup.py sucks when you have your library in a separate folder, adding these in to help setup envs
-
-# install from build directory
-.PHONY: install
-install: uninstall
-	cd $(BUILDDIR); pip install -e .
-
 # install from source tree for development
 .PHONY: install-dev
 install-dev: uninstall
@@ -182,30 +205,3 @@ install-dist: uninstall
 uninstall:
 	pip uninstall -y cellxgene || :
 
-
-# GUI
-
-.PHONY: build-assets
-build-assets:
-	pyside2-rcc server/gui/cellxgene.qrc -o server/gui/cellxgene_rc.py
-
-.PHONY: gui-spec-osx
-gui-spec-osx: clean-lite gui-env
-	pip install -e .[gui]
-	pyi-makespec -D -w --additional-hooks-dir server/gui/ -n cellxgene  --add-binary='/System/Library/Frameworks/Tk.framework/Tk':'tk' --add-binary='/System/Library/Frameworks/Tcl.framework/Tcl':'tcl'  --add-data server/app/web/templates/:server/app/web/templates/ --add-data server/app/web/static/:server/app/web/static/ --icon server/gui/images/cxg_icons.icns server/gui/main.py
-	mv cellxgene.spec cellxgene-osx.spec
-
-.PHONY: gui-spec-windows
-gui-spec-windows: clean-lite dev-env
-	pip install -e .[gui]
-	pyi-makespec -D -w --additional-hooks-dir server/gui/ -n cellxgene --add-data server/app/web/templates;server/app/web/templates --add-data server/app/web/static;server/app/web/static --icon server/gui/images/icon.ico server/gui/main.py
-	mv cellxgene.spec cellxgene-windows.spec
-
-.PHONY: gui-build-osx
-gui-build-osx: clean-lite
-	pyinstaller --clean cellxgene-osx.spec
-
-.PHONY: gui-build-windows
-gui-build-windows: clean-lite
-	pyinstaller --clean cellxgene-windows.spec
-

README.md

@@ -3,34 +3,41 @@
 _an interactive explorer for single-cell transcriptomics data_
 
 [![DOI](https://zenodo.org/badge/105615409.svg)](https://zenodo.org/badge/latestdoi/105615409) [![PyPI](https://img.shields.io/pypi/v/cellxgene)](https://pypi.org/project/cellxgene/) [![PyPI - Downloads](https://img.shields.io/pypi/dm/cellxgene)](https://pypistats.org/packages/cellxgene) [![GitHub last commit](https://img.shields.io/github/last-commit/chanzuckerberg/cellxgene)](https://github.com/chanzuckerberg/cellxgene/pulse)
+[![Push Tests](https://github.com/chanzuckerberg/cellxgene/workflows/Push%20Tests/badge.svg)](https://github.com/chanzuckerberg/cellxgene/actions?query=workflow%3A%22Push+Tests%22)
+[![Compatibility Tests](https://github.com/chanzuckerberg/cellxgene/workflows/Compatibility%20Tests/badge.svg)](https://github.com/chanzuckerberg/cellxgene/actions?query=workflow%3A%22Compatibility+Tests%22)
+![Code Coverage](https://codecov.io/gh/chanzuckerberg/cellxgene/branch/main/graph/badge.svg)
 
 cellxgene (pronounced "cell-by-gene") is an interactive data explorer for single-cell transcriptomics datasets, such as those coming from the [Human Cell Atlas](https://humancellatlas.org). Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data.
 
 Whether you need to visualize one thousand cells or one million, cellxgene helps you gain insight into your single-cell data.
 
-<img src="https://github.com/chanzuckerberg/cellxgene/raw/master/docs/images/crossfilter.gif" width="350" height="200" hspace="30"><img src="https://github.com/chanzuckerberg/cellxgene/raw/master/docs/images/category-breakdown.gif" width="350" height="200" hspace="30">
+<img src="https://github.com/chanzuckerberg/cellxgene/raw/main/docs/images/crossfilter.gif" width="350" height="200" hspace="30"><img src="https://github.com/chanzuckerberg/cellxgene/raw/main/docs/images/category-breakdown.gif" width="350" height="200" hspace="30">
 
 # Getting started
+
 ### The comprehensive guide to cellxgene
+
 [The cellxgene documentation is your one-stop-shop for information about cellxgene](https://chanzuckerberg.github.io/cellxgene/)! You may be particularly interested in:
-* Seeing [what cellxgene can do](https://chanzuckerberg.github.io/cellxgene/posts/gallery)  
-* Learning more about cellxgene [installation](https://chanzuckerberg.github.io/cellxgene/posts/install) and [usage](https://chanzuckerberg.github.io/cellxgene/posts/launch)
-* [Preparing your own data](https://chanzuckerberg.github.io/cellxgene/posts/prepare) for use in cellxgene
-* Checking out [our roadmap](https://chanzuckerberg.github.io/cellxgene/posts/roadmap) for future development
-* [Contributing](https://chanzuckerberg.github.io/cellxgene/posts/contribute) to cellxgene
+
+- Seeing [what cellxgene can do](https://chanzuckerberg.github.io/cellxgene/posts/gallery)
+- Learning more about cellxgene [installation](https://chanzuckerberg.github.io/cellxgene/posts/install) and [usage](https://chanzuckerberg.github.io/cellxgene/posts/launch)
+- [Preparing your own data](https://chanzuckerberg.github.io/cellxgene/posts/prepare) for use in cellxgene
+- Checking out [our roadmap](https://chanzuckerberg.github.io/cellxgene/posts/roadmap) for future development
+- [Contributing](https://chanzuckerberg.github.io/cellxgene/posts/contribute) to cellxgene
 
 ### Quick start
 
 To install cellxgene you need Python 3.6+. We recommend [installing cellxgene into a conda or virtual environment.](https://chanzuckerberg.github.io/cellxgene/posts/install)
 
 Install the package.
-``` bash
+
+```bash
 pip install cellxgene
 ```
 
 Launch cellxgene with an example [anndata](https://anndata.readthedocs.io/en/latest/) file
 
-``` bash
+```bash
 cellxgene launch https://cellxgene-example-data.czi.technology/pbmc3k.h5ad
 ```
 
@@ -38,6 +45,17 @@ To explore more datasets already formatted for cellxgene, check out the [Demo da
 see [Preparing your data](https://chanzuckerberg.github.io/cellxgene/posts/prepare) to learn more about formatting your own
 data for cellxgene.
 
+### Supported browsers
+
+cellxgene currently supports the following browsers:
+
+- Google Chrome 61+
+- Edge 15+
+- Firefox 60+
+- Safari 10.1+
+
+Please [file an issue](https://github.com/chanzuckerberg/cellxgene/issues/new/choose) if you would like us to add support for an unsupported browser.
+
 ### Finding help
 
 We'd love to hear from you!
@@ -48,6 +66,7 @@ For any errors, [report bugs on Github](https://github.com/chanzuckerberg/cellxg
 # Developing with cellxgene
 
 ### Contributing
+
 We warmly welcome contributions from the community! Please see our [contributing guide](https://chanzuckerberg.github.io/cellxgene/posts/contribute) and don't hesitate to open an issue or send a pull request to improve cellxgene.
 
 This project adheres to the Contributor Covenant [code of conduct](https://github.com/chanzuckerberg/.github/blob/master/CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to opensource@chanzuckerberg.com.
@@ -60,24 +79,9 @@ This project was started with the sole goal of empowering the scientific communi
 
 If you believe you have found a security issue, we would appreciate notification. Please send email to <security@chanzuckerberg.com>.
 
-# About
-### Core team
+# Inspiration
 
-The current core team:
-
-- Colin Megill, frontend & product design
-- Bruce Martin, software engineer
-- Sidney Bell, computational biologist
-- Lia Prins, designer
-- Severiano Badajoz, software engineer
-
-We would also like to gratefully acknowledge contributions from past core team members:
-
-- Charlotte Weaver, software engineer
-
-### Inspiration
-
-We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browswer](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [Gene Pattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data.
+We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browser](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [GenePattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data.
 
 We were inspired by Mike Bostock and the [crossfilter](https://github.com/crossfilter) team for the design of our filtering implementation.
 

app.json

@@ -1,21 +0,0 @@
-{
-  "name": "cellxgene",
-  "description": "An interactive explorer for single-cell transcriptomics data",
-  "repository": "https://github.com/chanzuckerberg/cellxgene/",
-  "logo": "https://cellxgene-example-data.czi.technology/favicon.png",
-  "keywords": [
-    "scientific",
-    "visualization",
-    "scrna-seq",
-    "transcriptomics",
-    "dataviz"
-  ],
-  "stack": "container",
-  "env": {
-    "DATASET": {
-      "description": "Link to dataset",
-      "value": "https://cellxgene-example-data.czi.technology/pbmc3k.h5ad",
-      "required": "true"
-    }
-  }
-}

cellxgene-osx.spec

@@ -1,41 +0,0 @@
-# -*- mode: python ; coding: utf-8 -*-
-
-block_cipher = None
-
-
-a = Analysis(['server/gui/main.py'],
-             pathex=['/Users/charlotteweaver/Documents/Git/cellxgene'],
-             binaries=[('/System/Library/Frameworks/Tk.framework/Tk', 'tk'), ('/System/Library/Frameworks/Tcl.framework/Tcl', 'tcl')],
-             datas=[('server/app/web/templates/', 'server/app/web/templates/'), ('server/app/web/static/', 'server/app/web/static/')],
-             hiddenimports=['sklearn', 'sklearn.utils._cython_blas', 'sklearn.neighbors.typedefs', 'sklearn.neighbors.quad_tree', 'sklearn.tree', 'sklearn.tree._utils'],
-             hookspath=['server/gui/'],
-             runtime_hooks=[],
-             excludes=[],
-             win_no_prefer_redirects=False,
-             win_private_assemblies=False,
-             cipher=block_cipher,
-             noarchive=False)
-pyz = PYZ(a.pure, a.zipped_data,
-             cipher=block_cipher)
-exe = EXE(pyz,
-          a.scripts,
-          [],
-          exclude_binaries=True,
-          name='cellxgene',
-          debug=False,
-          bootloader_ignore_signals=False,
-          strip=False,
-          upx=True,
-          console=False , icon='server/gui/images/cxg_icons.icns')
-coll = COLLECT(exe,
-               a.binaries,
-               a.zipfiles,
-               a.datas,
-               strip=False,
-               upx=True,
-               upx_exclude=[],
-               name='cellxgene')
-app = BUNDLE(coll,
-             name='cellxgene.app',
-             icon='server/gui/images/cxg_icons.icns',
-             bundle_identifier=None)

cellxgene-windows.spec

@@ -1,36 +0,0 @@
-# -*- mode: python -*-
-
-block_cipher = None
-
-
-a = Analysis(['server\\gui\\main.py'],
-             pathex=['C:\\Users\\Charlotte\\Documents\\git\\cellxgene'],
-             binaries=[],
-             datas=[('server/app/web/templates/', 'server/app/web/templates'), ('server/app/web/static/', 'server/app/web/static')],
-             hiddenimports=[],
-             hookspath=['server/gui/'],
-             runtime_hooks=[],
-             excludes=[],
-             win_no_prefer_redirects=False,
-             win_private_assemblies=False,
-             cipher=block_cipher,
-             noarchive=False)
-pyz = PYZ(a.pure, a.zipped_data,
-             cipher=block_cipher)
-exe = EXE(pyz,
-          a.scripts,
-          [],
-          exclude_binaries=True,
-          name='cellxgene',
-          debug=False,
-          bootloader_ignore_signals=False,
-          strip=False,
-          upx=True,
-          console=False , icon='server\\gui\\images\\icon.ico')
-coll = COLLECT(exe,
-               a.binaries,
-               a.zipfiles,
-               a.datas,
-               strip=False,
-               upx=True,
-               name='cellxgene')

client/Makefile

@@ -1,20 +1,57 @@
+include ../common.mk
+
+ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../server/test/fixtures/pbmc3k-annotations.csv)
+ANNOTATIONS_FILENAME := $(shell basename $(ANNOTATIONS))
+
+CXG_CONFIG := $(if $(CXG_CONFIG), $(CXG_CONFIG), ./__tests__/e2e/test_config.yaml)
+
+# Packaging
 .PHONY: clean
 clean:
 	rm -rf node_modules
+	rm -f __tests__/screenshots/*.png
 
 .PHONY: ci
 ci:
-	npm ci client
+	npm ci
 
 .PHONY: install
 install:
-	npm install client
+	npm install
 
 .PHONY: build
+WEBPACK_CONFIG ?= configuration/webpack/webpack.config.prod.js
 build:
-	npm run build
+	npm run build $(WEBPACK_CONFIG)
+
+# Development convenience methods
+.PHONY: start-frontend
+start-frontend:
+	node server/development.js
+
+# start an instance of cellxgene and run the end-to-end tests
+.PHONY: smoke-test
+smoke-test:
+	start_server_and_test \
+		'CXG_OPTIONS="--config-file $(CXG_CONFIG)" $(MAKE) start-server' \
+		$(CXG_SERVER_PORT) \
+		'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" CXG_AUTH_TYPE="test" npm run e2e -- --verbose false'
+
+# start an instance of cellxgene and run the end-to-end annotations tests
+.PHONY: smoke-test-annotations
+smoke-test-annotations:
+	$(eval TMP_DIR := $(shell  mktemp -d /tmp/cellxgene_XXXXXX))
+	cp $(ANNOTATIONS) $(TMP_DIR)/ && \
+	start_server_and_test \
+		'CXG_OPTIONS="--annotations-file $(TMP_DIR)/$(ANNOTATIONS_FILENAME)" $(MAKE) start-server' \
+		$(CXG_SERVER_PORT) \
+		'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" npm run e2e-annotations -- --verbose false'
+	rm -rf $(TMP_DIR)
+
+.PHONY: unit-test
+unit-test:
+	node node_modules/jest/bin/jest.js --testPathIgnorePatterns e2e
 
 # pass remaining commands through to npm run
 %:
 	npm run $(*)
-

client/__tests__/e2e/__snapshots__/e2e.test.js.snap

@@ -0,0 +1,5 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`did launch page launched 1`] = `"<span style=\\"max-width: 155px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">pbm</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">c3k</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">c3k</span></span></span>"`;
+
+exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testclass=\\"category-expand\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"max-width: 265px;\\"><span data-testid=\\"louvain:category-label\\" aria-label=\\"louvain\\" class=\\"\\" tabindex=\\"0\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">vain</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">vain</span></span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><desc>tint</desc><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"></div><div></div>"`;

client/__tests__/e2e/__snapshots__/e2eAnnotations.test.js.snap

@@ -0,0 +1,23 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`annotations stacked bar graph renders 1`] = `
+Array [
+  "<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-TEST-LABEL\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value\\" aria-label=\\"TEST-LABEL\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">TEST-</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">LABEL</span><span style=\\"position: absolute; right: 0px; color: black;\\">LABEL</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-TEST-LABEL\\" style=\\"color: black;\\">0</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:TEST-LABEL:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
+  "<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-unassigned\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value\\" aria-label=\\"unassigned\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: rgb(171, 171, 171); font-style: italic; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">unass</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">igned</span><span style=\\"position: absolute; right: 0px; color: rgb(171, 171, 171);\\">igned</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"><canvas width=\\"100\\" height=\\"11\\" style=\\"margin-right: 5px; width: 100px; height: 11px;\\"></canvas></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-unassigned\\" style=\\"color: rgb(171, 171, 171); font-style: italic;\\">2133</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:unassigned:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
+]
+`;
+
+exports[`annotations stacked bar graph renders 2`] = `
+Array [
+  "<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-TEST-LABEL\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value\\" aria-label=\\"TEST-LABEL\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">TEST-</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">LABEL</span><span style=\\"position: absolute; right: 0px; color: black;\\">LABEL</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-TEST-LABEL\\" style=\\"color: black;\\">0</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:TEST-LABEL:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
+  "<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-unassigned\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value\\" aria-label=\\"unassigned\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: rgb(171, 171, 171); font-style: italic; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">unass</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">igned</span><span style=\\"position: absolute; right: 0px; color: rgb(171, 171, 171);\\">igned</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"><canvas width=\\"100\\" height=\\"11\\" style=\\"margin-right: 5px; width: 100px; height: 11px;\\"></canvas></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-unassigned\\" style=\\"color: rgb(171, 171, 171); font-style: italic;\\">2638</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:unassigned:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
+]
+`;
+
+exports[`annotations truncate midpoint whitespace 1`] = `"<span data-testid=\\"categorical-value-TEST-CATEGORY-123 456\\" data-testclass=\\"categorical-value\\" aria-label=\\"123 456\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 187px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">123</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">&nbsp;456</span><span style=\\"position: absolute; right: 0px; color: black;\\">&nbsp;456</span></span></span></span>"`;
+
+exports[`annotations truncate midpoint whitespace 2`] = `"<span data-testid=\\"categorical-value-TEST-CATEGORY-123 456\\" data-testclass=\\"categorical-value\\" aria-label=\\"123 456\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 187px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">123</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">&nbsp;456</span><span style=\\"position: absolute; right: 0px; color: black;\\">&nbsp;456</span></span></span></span>"`;
+
+exports[`annotations truncate single character 1`] = `"<span data-testid=\\"categorical-value-TEST-CATEGORY-T\\" data-testclass=\\"categorical-value\\" aria-label=\\"T\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 187px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">T</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\"></span><span style=\\"position: absolute; right: 0px; color: black;\\"></span></span></span></span>"`;
+
+exports[`annotations truncate single character 2`] = `"<span data-testid=\\"categorical-value-TEST-CATEGORY-T\\" data-testclass=\\"categorical-value\\" aria-label=\\"T\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 187px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">T</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\"></span><span style=\\"position: absolute; right: 0px; color: black;\\"></span></span></span></span>"`;

client/__tests__/e2e/cellxgeneActions.js

@@ -0,0 +1,388 @@
+/* eslint-disable no-await-in-loop -- await in loop is needed to emulate sequential user actions  */
+import { strict as assert } from "assert";
+import {
+  clearInputAndTypeInto,
+  clickOn,
+  getAllByClass,
+  getOneElementInnerText,
+  typeInto,
+  waitByID,
+  waitByClass,
+  waitForAllByIds,
+  clickOnUntil,
+  getTestClass,
+  getTestId,
+  isElementPresent,
+  goToPage,
+} from "./puppeteerUtils";
+
+import { appUrlBase } from "./config";
+
+export async function drag(testId, start, end, lasso = false) {
+  const layout = await waitByID(testId);
+  const elBox = await layout.boxModel();
+  const x1 = elBox.content[0].x + start.x;
+  const x2 = elBox.content[0].x + end.x;
+  const y1 = elBox.content[0].y + start.y;
+  const y2 = elBox.content[0].y + end.y;
+  await page.mouse.move(x1, y1);
+  await page.mouse.down();
+  if (lasso) {
+    await page.mouse.move(x2, y1);
+    await page.mouse.move(x2, y2);
+    await page.mouse.move(x1, y2);
+    await page.mouse.move(x1, y1);
+  } else {
+    await page.mouse.move(x2, y2);
+  }
+  await page.mouse.up();
+}
+
+export async function clickOnCoordinate(testId, coord) {
+  const layout = await expect(page).toMatchElement(getTestId(testId));
+  const elBox = await layout.boxModel();
+
+  if (!elBox) {
+    throw Error("Layout's boxModel is not available!");
+  }
+
+  const x = elBox.content[0].x + coord.x;
+  const y = elBox.content[0].y + coord.y;
+  await page.mouse.click(x, y);
+}
+
+export async function getAllHistograms(testclass, testIds) {
+  const histTestIds = testIds.map((tid) => `histogram-${tid}`);
+
+  // these load asynchronously, so we need to wait for each histogram individually,
+  // and they may be quite slow in some cases.
+  await waitForAllByIds(histTestIds, { timeout: 4 * 60 * 1000 });
+
+  const allHistograms = await getAllByClass(testclass);
+
+  const testIDs = await Promise.all(
+    allHistograms.map((hist) => {
+      return page.evaluate((elem) => {
+        return elem.dataset.testid;
+      }, hist);
+    })
+  );
+
+  return testIDs.map((id) => id.replace(/^histogram-/, ""));
+}
+
+export async function getAllCategoriesAndCounts(category) {
+  // these load asynchronously, so we have to wait for the specific category.
+  await waitByID(`category-${category}`);
+
+  return page.$$eval(
+    `[data-testid="category-${category}"] [data-testclass='categorical-row']`,
+    (rows) =>
+      Object.fromEntries(
+        rows.map((row) => {
+          const cat = row
+            .querySelector("[data-testclass='categorical-value']")
+            .getAttribute("aria-label");
+
+          const count = row.querySelector(
+            "[data-testclass='categorical-value-count']"
+          ).innerText;
+
+          return [cat, count];
+        })
+      )
+  );
+}
+
+export async function getCellSetCount(num) {
+  await clickOn(`cellset-button-${num}`);
+  return getOneElementInnerText(`[data-testid='cellset-count-${num}']`);
+}
+
+export async function resetCategory(category) {
+  const checkboxId = `${category}:category-select`;
+  await waitByID(checkboxId);
+  const checkedPseudoclass = await page.$eval(
+    `[data-testid='${checkboxId}']`,
+    (el) => el.matches(":checked")
+  );
+  if (!checkedPseudoclass) await clickOn(checkboxId);
+
+  const categoryRow = await waitByID(`${category}:category-expand`);
+
+  const isExpanded = await categoryRow.$(
+    "[data-testclass='category-expand-is-expanded']"
+  );
+
+  if (isExpanded) await clickOn(`${category}:category-expand`);
+}
+
+export async function calcCoordinate(testId, xAsPercent, yAsPercent) {
+  const el = await waitByID(testId);
+  const size = await el.boxModel();
+  return {
+    x: Math.floor(size.width * xAsPercent),
+    y: Math.floor(size.height * yAsPercent),
+  };
+}
+
+export async function calcDragCoordinates(testId, coordinateAsPercent) {
+  return {
+    start: await calcCoordinate(
+      testId,
+      coordinateAsPercent.x1,
+      coordinateAsPercent.y1
+    ),
+    end: await calcCoordinate(
+      testId,
+      coordinateAsPercent.x2,
+      coordinateAsPercent.y2
+    ),
+  };
+}
+
+export async function selectCategory(category, values, reset = true) {
+  if (reset) await resetCategory(category);
+
+  await clickOn(`${category}:category-expand`);
+  await clickOn(`${category}:category-select`);
+
+  for (const value of values) {
+    await clickOn(`categorical-value-select-${category}-${value}`);
+  }
+}
+
+export async function expandCategory(category) {
+  const expand = await waitByID(`${category}:category-expand`);
+  const notExpanded = await expand.$(
+    "[data-testclass='category-expand-is-not-expanded']"
+  );
+  if (notExpanded) await clickOn(`${category}:category-expand`);
+}
+
+export async function clip(min = 0, max = 100) {
+  await clickOn("visualization-settings");
+  await clearInputAndTypeInto("clip-min-input", min);
+  await clearInputAndTypeInto("clip-max-input", max);
+  await clickOn("clip-commit");
+}
+
+export async function createCategory(categoryName) {
+  await clickOnUntil("open-annotation-dialog", async () => {
+    await expect(page).toMatchElement(getTestId("new-category-name"));
+  });
+
+  await typeInto("new-category-name", categoryName);
+  await clickOn("submit-category");
+}
+
+export async function duplicateCategory(categoryName) {
+  await clickOn("open-annotation-dialog");
+
+  await typeInto("new-category-name", categoryName);
+
+  const dropdownOptionClass = "duplicate-category-dropdown-option";
+
+  await clickOnUntil("duplicate-category-dropdown", async () => {
+    await expect(page).toMatchElement(getTestClass(dropdownOptionClass));
+  });
+
+  const option = await expect(page).toMatchElement(
+    getTestClass(dropdownOptionClass)
+  );
+
+  await option.click();
+
+  await clickOnUntil("submit-category", async () => {
+    await expect(page).toMatchElement(
+      getTestId(`${categoryName}:category-expand`)
+    );
+  });
+
+  await waitByClass("autosave-complete");
+}
+
+export async function renameCategory(oldCategoryName, newCategoryName) {
+  await clickOn(`${oldCategoryName}:see-actions`);
+  await clickOn(`${oldCategoryName}:edit-category-mode`);
+  await clearInputAndTypeInto(
+    `${oldCategoryName}:edit-category-name-text`,
+    newCategoryName
+  );
+  await clickOn(`${oldCategoryName}:submit-category-edit`);
+}
+
+export async function deleteCategory(categoryName) {
+  const targetId = `${categoryName}:delete-category`;
+
+  await clickOnUntil(`${categoryName}:see-actions`, async () => {
+    await expect(page).toMatchElement(getTestId(targetId));
+  });
+
+  await clickOn(targetId);
+
+  await assertCategoryDoesNotExist();
+}
+
+export async function createLabel(categoryName, labelName) {
+  /**
+   * (thuang): This explicit wait is needed, since currently showing
+   * the modal again quickly after the previous action dismissing the
+   * modal will persist the input value from the previous action.
+   *
+   * To reproduce:
+   * 1. Click on the plus sign to show the modal to add a new label to the category
+   * 2. Type `123` in the input box
+   * 3. Hover over your mouse over the plus sign and double click to quickly dismiss and
+   * invoke the modal again
+   * 4. You will see `123` is persisted in the input box
+   * 5. Expected behavior is to get an empty input box
+   */
+  await page.waitFor(500);
+
+  await clickOn(`${categoryName}:see-actions`);
+
+  await clickOn(`${categoryName}:add-new-label-to-category`);
+
+  await typeInto(`${categoryName}:new-label-name`, labelName);
+
+  await clickOn(`${categoryName}:submit-label`);
+}
+
+export async function deleteLabel(categoryName, labelName) {
+  await expandCategory(categoryName);
+  await clickOn(`${categoryName}:${labelName}:see-actions`);
+  await clickOn(`${categoryName}:${labelName}:delete-label`);
+}
+
+export async function renameLabel(categoryName, oldLabelName, newLabelName) {
+  await expandCategory(categoryName);
+  await clickOn(`${categoryName}:${oldLabelName}:see-actions`);
+  await clickOn(`${categoryName}:${oldLabelName}:edit-label`);
+  await clearInputAndTypeInto(
+    `${categoryName}:${oldLabelName}:edit-label-name`,
+    newLabelName
+  );
+  await clickOn(`${categoryName}:${oldLabelName}:submit-label-edit`);
+}
+
+export async function addGeneToSearch(geneName) {
+  await typeInto("gene-search", geneName);
+  await page.keyboard.press("Enter");
+  await page.waitForSelector(`[data-testid='histogram-${geneName}']`);
+}
+
+export async function subset(coordinatesAsPercent) {
+  // In order to deselect the selection after the subset, make sure we have some clear part
+  // of the scatterplot we can click on
+  assert(coordinatesAsPercent.x2 < 0.99 || coordinatesAsPercent.y2 < 0.99);
+  const lassoSelection = await calcDragCoordinates(
+    "layout-graph",
+    coordinatesAsPercent
+  );
+  await drag("layout-graph", lassoSelection.start, lassoSelection.end, true);
+  await clickOn("subset-button");
+  const clearCoordinate = await calcCoordinate("layout-graph", 0.5, 0.99);
+  await clickOnCoordinate("layout-graph", clearCoordinate);
+}
+
+export async function setSellSet(cellSet, cellSetNum) {
+  const selections = cellSet.filter((sel) => sel.kind === "categorical");
+
+  for (const selection of selections) {
+    await selectCategory(selection.metadata, selection.values, true);
+  }
+
+  await getCellSetCount(cellSetNum);
+}
+
+export async function runDiffExp(cellSet1, cellSet2) {
+  await setSellSet(cellSet1, 1);
+  await setSellSet(cellSet2, 2);
+  await clickOn("diffexp-button");
+}
+
+export async function bulkAddGenes(geneNames) {
+  await clickOn("section-bulk-add");
+  await typeInto("input-bulk-add", geneNames.join(","));
+  await page.keyboard.press("Enter");
+}
+
+export async function assertCategoryDoesNotExist(categoryName) {
+  const result = await isElementPresent(
+    getTestId(`${categoryName}:category-label`)
+  );
+
+  await expect(result).toBe(false);
+}
+
+export async function login() {
+  const email = `cellxgene-smoke-test+${process.env.DEPLOYMENT_STAGE}@chanzuckerberg.com`;
+  const password = "Test1111";
+
+  await goToPage(appUrlBase);
+
+  await clickOn("log-in");
+
+  // (thuang): Auth0 form is unstable and unsafe for input until verified
+  await waitUntilFormFieldStable('[name="email"]');
+
+  await expect(page).toFillForm("form", {
+    email,
+    password,
+  });
+
+  await Promise.all([
+    page.waitForNavigation({ waitUntil: "networkidle0" }),
+    expect(page).toClick('[name="submit"]'),
+  ]);
+
+  expect(page.url()).toContain(appUrlBase);
+}
+
+export async function logout() {
+  await clickOnUntil("user-info", async () => {
+    await waitByID("log-out");
+    await Promise.all([
+      page.waitForNavigation({ waitUntil: "networkidle0" }),
+      clickOn("log-out"),
+    ]);
+  });
+
+  await waitByID("log-in");
+}
+
+async function waitUntilFormFieldStable(selector) {
+  const MAX_RETRY = 10;
+  const WAIT_FOR_MS = 200;
+
+  const EXPECTED_VALUE = "aaa";
+
+  let retry = 0;
+
+  while (retry < MAX_RETRY) {
+    try {
+      await expect(page).toFill(selector, EXPECTED_VALUE);
+
+      const fieldHandle = await expect(page).toMatchElement(selector);
+
+      const fieldValue = await page.evaluate(
+        (input) => input.value,
+        fieldHandle
+      );
+
+      expect(fieldValue).toBe(EXPECTED_VALUE);
+
+      break;
+    } catch (error) {
+      retry += 1;
+
+      await page.waitFor(WAIT_FOR_MS);
+    }
+  }
+
+  if (retry === MAX_RETRY) {
+    throw Error("clickOnUntil() assertion failed!");
+  }
+}
+/* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/config.js

@@ -1,6 +1,8 @@
-export const jest_env = process.env.JEST_ENV || "dev";
-export const appPort = process.env.JEST_CXG_PORT || 3000;
-export const appUrlBase = `http://localhost:${appPort}`;
-export const DEV = jest_env === "dev";
-export const DEBUG = jest_env === "debug";
+import * as ENV_DEFAULT from "../../../environment.default.json";
+
+export const jestEnv = process.env.JEST_ENV || ENV_DEFAULT.JEST_ENV;
+export const appUrlBase =
+  process.env.CXG_URL_BASE || `http://localhost:${ENV_DEFAULT.CXG_CLIENT_PORT}`;
 export const DATASET = "pbmc3k";
+export const isDev = jestEnv === ENV_DEFAULT.DEV;
+export const isDebug = jestEnv === ENV_DEFAULT.DEBUG;

client/__tests__/e2e/data.js

@@ -4,7 +4,7 @@ export const datasets = {
     dataframe: {
       nObs: "2638",
       nVar: "1838",
-      type: "float32"
+      type: "float32",
     },
     categorical: {
       louvain: {
@@ -15,47 +15,47 @@ export const datasets = {
         "Dendritic cells": "37",
         "FCGR3A+ Monocytes": "150",
         Megakaryocytes: "15",
-        "NK cells": "154"
-      }
+        "NK cells": "154",
+      },
     },
     continuous: {
       n_genes: "int32",
       percent_mito: "float32",
-      n_counts: "float32"
+      n_counts: "float32",
     },
     cellsets: {
       lasso: [
         {
           "coordinates-as-percent": { x1: 0.1, y1: 0.25, x2: 0.7, y2: 0.75 },
-          count: "1181"
-        }
+          count: "1131",
+        },
       ],
       categorical: [
         {
           metadata: "louvain",
           values: ["B cells", "Megakaryocytes"],
-          count: "357"
-        }
+          count: "357",
+        },
       ],
       continuous: [
         {
           metadata: "n_genes",
           "coordinates-as-percent": { x1: 0.25, y1: 0.5, x2: 0.55, y2: 0.5 },
-          count: "1552"
-        }
-      ]
+          count: "1537",
+        },
+      ],
     },
 
     diffexp: {
       cellset1: [
-        { kind: "categorical", metadata: "louvain", values: ["B cells"] }
+        { kind: "categorical", metadata: "louvain", values: ["B cells"] },
       ],
       cellset2: [
         {
           kind: "categorical",
           metadata: "louvain",
-          values: ["CD4 T cells", "NK cells"]
-        }
+          values: ["CD4 T cells", "NK cells"],
+        },
       ],
       "gene-results": [
         "HLA-DRB1",
@@ -67,21 +67,21 @@ export const datasets = {
         "HLA-DQB1",
         "MS4A1",
         "IL32",
-        "CD37"
-      ]
+        "CD37",
+      ],
     },
 
     genes: {
       bulkadd: ["S100A8", "FCGR3A", "LGALS2", "GSTP1"],
-      search: "ACD"
+      search: "ACD",
     },
     subset: {
       cellset1: [
         {
           kind: "categorical",
           metadata: "louvain",
-          values: ["B cells", "Megakaryocytes"]
-        }
+          values: ["B cells", "Megakaryocytes"],
+        },
       ],
       count: "357",
       categorical: {
@@ -93,27 +93,46 @@ export const datasets = {
           "Dendritic cells": "0",
           "FCGR3A+ Monocytes": "0",
           Megakaryocytes: "15",
-          "NK cells": "0"
-        }
+          "NK cells": "0",
+        },
       },
       lasso: {
         "coordinates-as-percent": { x1: 0.25, y1: 0.05, x2: 0.75, y2: 0.55 },
-        count: "329"
-      }
+        count: "331",
+      },
     },
     scatter: {
-      genes: { x: "S100A8", y: "FCGR3A" }
+      genes: { x: "S100A8", y: "FCGR3A" },
     },
     pan: {
-      "coordinates-as-percent": { x1: 0.75, y1: 0.75, x2: 0.35, y2: 0.35 }
+      "coordinates-as-percent": { x1: 0.75, y1: 0.75, x2: 0.35, y2: 0.35 },
     },
     features: {
       panzoom: {
         lasso: {
           "coordinates-as-percent": { x1: 0.3, y1: 0.3, x2: 0.5, y2: 0.5 },
-          count: "24"
-        }
-      }
+          count: "38",
+        },
+      },
+    },
+    categoryLabel: {
+      lasso: {
+        "coordinates-as-percent": { x1: 0.05, y1: 0.3, x2: 0.5, y2: 0.5 },
+      },
+      newCount: {
+        bySubsetConfig: {
+          false: "668",
+          true: "659",
+        },
+      },
+    },
+    annotationsFromFile: {
+      count: {
+        bySubsetConfig: {
+          false: "1161",
+          true: "852",
+        },
+      },
     },
     clip: {
       min: "30",
@@ -121,8 +140,8 @@ export const datasets = {
       metadata: "n_genes",
       gene: "S100A8",
       "coordinates-as-percent": { x1: 0.25, y1: 0.5, x2: 0.55, y2: 0.5 },
-      count: "392",
-      "gene-cell-count": "421"
-    }
-  }
+      count: "386",
+      "gene-cell-count": "416",
+    },
+  },
 };

client/__tests__/e2e/e2e.test.js

@@ -1,79 +1,72 @@
-/*
-Smoke test suite that will be run in Travis CI
-
-Tests included in this file are expected to be relatively stable and test core features
+/**
+ * Smoke test suite that will be run in Travis CI
+ * Tests included in this file are expected to be relatively stable and test core features
  */
-import puppeteer from "puppeteer";
-import { appUrlBase, DEBUG, DEV, DATASET } from "./config";
-import { puppeteerUtils, cellxgeneActions } from "./puppeteerUtils";
+
+/* eslint-disable no-await-in-loop -- await in loop is needed to emulate sequential user actions  */
+import { appUrlBase, DATASET } from "./config";
+
 import { datasets } from "./data";
 
-let browser, page, utils, cxgActions, spy;
-const browserViewport = { width: 1280, height: 960 };
-let data = datasets[DATASET];
+import {
+  clickOn,
+  getAllByClass,
+  getElementCoordinates,
+  getOneElementInnerHTML,
+  getTestId,
+  goToPage,
+  typeInto,
+  waitByID,
+  clickOnUntil,
+} from "./puppeteerUtils";
 
-if (DEBUG) jest.setTimeout(100000);
-if (DEV) jest.setTimeout(10000);
+import {
+  addGeneToSearch,
+  bulkAddGenes,
+  calcDragCoordinates,
+  clip,
+  drag,
+  getAllCategoriesAndCounts,
+  getAllHistograms,
+  getCellSetCount,
+  runDiffExp,
+  selectCategory,
+  subset,
+  login,
+  logout,
+} from "./cellxgeneActions";
 
-beforeAll(async () => {
-  const browserParams = DEV
-    ? { headless: false, slowMo: 5 }
-    : DEBUG
-    ? { headless: false, slowMo: 100, devtools: true }
-    : {};
-  browser = await puppeteer.launch(browserParams);
-  page = await browser.newPage();
-  await page.setViewport(browserViewport);
-  if (DEV || DEBUG) {
-    page.on("console", async msg => {
-      // If there is a console.error but an error is not thrown, this will ensure the test fails
-      if (msg.type() === "error") {
-        const errorMsgText = await Promise.all(
-          // TODO can we do this without internal properties?
-          msg.args().map(arg => arg._remoteObject.description)
-        );
-        throw new Error(`Console error: ${errorMsgText}`);
-      }
-      console.log(`PAGE LOG: ${msg.text()}`);
-    });
-  }
-  page.on("pageerror", err => {
-    throw new Error(`Console error: ${err}`);
-  });
-  utils = puppeteerUtils(page);
-  cxgActions = cellxgeneActions(page);
-});
-
-beforeEach(async () => {
-  await page.goto(appUrlBase);
-});
-
-afterAll(() => {
-  if (!DEBUG) {
-    browser.close();
-  }
-});
+const data = datasets[DATASET];
 
 describe("did launch", () => {
   test("page launched", async () => {
-    let el = await utils.getOneElementInnerHTML("[data-testid='header']");
-    expect(el).toBe(data.title);
+    await goToPage(appUrlBase);
+
+    const element = await getOneElementInnerHTML(getTestId("header"));
+
+    expect(element).toMatchSnapshot();
   });
 });
 
 describe("metadata loads", () => {
   test("categories and values from dataset appear", async () => {
-    for (const label in data.categorical) {
-      await utils.waitByID(`category-${label}`);
-      const categoryName = await utils.getOneElementInnerText(
-        `[data-testid="category-${label}"]`
+    await goToPage(appUrlBase);
+
+    for (const label of Object.keys(data.categorical)) {
+      const element = await getOneElementInnerHTML(
+        getTestId(`category-${label}`)
       );
-      expect(categoryName).toMatch(label);
-      await utils.clickOn(`category-expand-${label}`);
-      const categories = await cxgActions.getAllCategoriesAndCounts(label);
+
+      expect(element).toMatchSnapshot();
+
+      await clickOn(`${label}:category-expand`);
+
+      const categories = await getAllCategoriesAndCounts(label);
+
       expect(Object.keys(categories)).toMatchObject(
         Object.keys(data.categorical[label])
       );
+
       expect(Object.values(categories)).toMatchObject(
         Object.values(data.categorical[label])
       );
@@ -81,58 +74,76 @@ describe("metadata loads", () => {
   });
 
   test("continuous data appears", async () => {
-    for (const label in data.continuous) {
-      await utils.waitByID(`histogram-${label}`);
+    await goToPage(appUrlBase);
+
+    for (const label of Object.keys(data.continuous)) {
+      await waitByID(`histogram-${label}`);
     }
   });
 });
 
 describe("cell selection", () => {
   test("selects all cells cellset 1", async () => {
-    const cellCount = await cxgActions.cellSet(1);
+    await goToPage(appUrlBase);
+
+    const cellCount = await getCellSetCount(1);
     expect(cellCount).toBe(data.dataframe.nObs);
   });
 
   test("selects all cells cellset 2", async () => {
-    const cellCount = await cxgActions.cellSet(2);
+    await goToPage(appUrlBase);
+
+    const cellCount = await getCellSetCount(2);
     expect(cellCount).toBe(data.dataframe.nObs);
   });
 
   test("selects cells via lasso", async () => {
+    await goToPage(appUrlBase);
+
     for (const cellset of data.cellsets.lasso) {
-      const cellset1 = await cxgActions.calcDragCoordinates(
+      const cellset1 = await calcDragCoordinates(
         "layout-graph",
         cellset["coordinates-as-percent"]
       );
-      await cxgActions.drag("layout-graph", cellset1.start, cellset1.end, true);
-      const cellCount = await cxgActions.cellSet(1);
+
+      await drag("layout-graph", cellset1.start, cellset1.end, true);
+      const cellCount = await getCellSetCount(1);
       expect(cellCount).toBe(cellset.count);
     }
   });
 
   test("selects cells via categorical", async () => {
+    await goToPage(appUrlBase);
+
     for (const cellset of data.cellsets.categorical) {
-      await utils.clickOn(`category-expand-${cellset.metadata}`);
-      await utils.clickOn(`category-select-${cellset.metadata}`);
-      for (const val of cellset.values) {
-        await utils.clickOn(
-          `categorical-value-select-${cellset.metadata}-${val}`
-        );
+      await clickOn(`${cellset.metadata}:category-expand`);
+      await clickOn(`${cellset.metadata}:category-select`);
+
+      for (const value of cellset.values) {
+        await clickOn(`categorical-value-select-${cellset.metadata}-${value}`);
       }
-      const cellCount = await cxgActions.cellSet(1);
+
+      const cellCount = await getCellSetCount(1);
+
       expect(cellCount).toBe(cellset.count);
     }
   });
 
   test("selects cells via continuous", async () => {
+    await goToPage(appUrlBase);
+
     for (const cellset of data.cellsets.continuous) {
       const histBrushableAreaId = `histogram-${cellset.metadata}-plot-brushable-area`;
-      const coords = await cxgActions.calcDragCoordinates(
+
+      const coords = await calcDragCoordinates(
         histBrushableAreaId,
         cellset["coordinates-as-percent"]
       );
-      await cxgActions.drag(histBrushableAreaId, coords.start, coords.end);
-      const cellCount = await cxgActions.cellSet(1);
+
+      await drag(histBrushableAreaId, coords.start, coords.end);
+
+      const cellCount = await getCellSetCount(1);
+
       expect(cellCount).toBe(cellset.count);
     }
   });
@@ -140,168 +151,217 @@ describe("cell selection", () => {
 
 describe("gene entry", () => {
   test("search for single gene", async () => {
-    // blueprint's  typeahead is treating typing weird, clicking & waiting first solves this
-    await utils.typeInto("gene-search", data.genes.search);
-    await page.keyboard.press("Enter");
-    await page.waitForSelector(
-      `[data-testid='histogram-${data.genes.search}']`
-    );
+    await goToPage(appUrlBase);
+    await addGeneToSearch(data.genes.search);
   });
 
   test("bulk add genes", async () => {
-    await cxgActions.reset();
-    const testGenes = data.genes.bulkadd;
-    await utils.clickOn("section-bulk-add");
-    await utils.typeInto("input-bulk-add", testGenes.join(","));
-    await page.keyboard.press("Enter");
+    await goToPage(appUrlBase);
 
-    const allHistograms = await cxgActions.getAllHistograms(
+    const testGenes = data.genes.bulkadd;
+
+    await bulkAddGenes(testGenes);
+    const allHistograms = await getAllHistograms(
       "histogram-user-gene",
       testGenes
     );
+
     expect(allHistograms).toEqual(expect.arrayContaining(testGenes));
-    expect(allHistograms.length).toEqual(testGenes.length);
+    expect(allHistograms).toHaveLength(testGenes.length);
   });
 });
 
-describe("diffexp", () => {
+describe("differential expression", () => {
   test("selects cells, saves them and performs diffexp", async () => {
-    for (const select of data.diffexp.cellset1) {
-      if (select.kind === "categorical") {
-        await cxgActions.selectCategory(select.metadata, select.values, true);
-      }
-    }
-    await cxgActions.cellSet(1);
-    for (const select of data.diffexp.cellset2) {
-      if (select.kind === "categorical") {
-        await cxgActions.selectCategory(select.metadata, select.values, true);
-      }
-    }
-    await cxgActions.cellSet(2);
-    await utils.clickOn("diffexp-button");
-    const allHistograms = await cxgActions.getAllHistograms(
+    await goToPage(appUrlBase);
+
+    await runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
+
+    const allHistograms = await getAllHistograms(
       "histogram-diffexp",
       data.diffexp["gene-results"]
     );
+
     expect(allHistograms).toEqual(
       expect.arrayContaining(data.diffexp["gene-results"])
     );
-    expect(allHistograms.length).toEqual(data.diffexp["gene-results"].length);
+
+    expect(allHistograms).toHaveLength(data.diffexp["gene-results"].length);
   });
 });
 
-describe("subset/reset", () => {
+describe("subset", () => {
   test("subset - cell count matches", async () => {
-    for (const select of data.subset.cellset1) {
-      if (select.kind === "categorical") {
-        await cxgActions.selectCategory(select.metadata, select.values, true);
-      }
-    }
-    await utils.clickOn("subset-button");
-    for (const label in data.subset.categorical) {
-      const categories = await cxgActions.getAllCategoriesAndCounts(label);
-      expect(Object.keys(categories)).toMatchObject(
-        Object.keys(data.subset.categorical[label])
-      );
-      expect(Object.values(categories)).toMatchObject(
-        Object.values(data.subset.categorical[label])
-      );
-    }
-  });
+    await goToPage(appUrlBase);
 
-  test("reset after subset", async () => {
     for (const select of data.subset.cellset1) {
       if (select.kind === "categorical") {
-        await cxgActions.selectCategory(select.metadata, select.values, true);
+        await selectCategory(select.metadata, select.values, true);
       }
     }
-    await utils.clickOn("subset-button");
-    for (const label in data.subset.categorical) {
-      const categories = await cxgActions.getAllCategoriesAndCounts(label);
+
+    await clickOn("subset-button");
+
+    for (const label of Object.keys(data.subset.categorical)) {
+      const categories = await getAllCategoriesAndCounts(label);
+
       expect(Object.keys(categories)).toMatchObject(
         Object.keys(data.subset.categorical[label])
       );
+
       expect(Object.values(categories)).toMatchObject(
         Object.values(data.subset.categorical[label])
       );
     }
-    await cxgActions.reset();
-    for (const label in data.categorical) {
-      await utils.waitByID(`category-${label}`);
-      const categoryName = await utils.getOneElementInnerText(
-        `[data-testid="category-${label}"]`
-      );
-      expect(categoryName).toMatch(label);
-      const categories = await cxgActions.getAllCategoriesAndCounts(label);
-      expect(Object.keys(categories)).toMatchObject(
-        Object.keys(data.categorical[label])
-      );
-      expect(Object.values(categories)).toMatchObject(
-        Object.values(data.categorical[label])
-      );
-    }
   });
 
   test("lasso after subset", async () => {
+    await goToPage(appUrlBase);
+
     for (const select of data.subset.cellset1) {
       if (select.kind === "categorical") {
-        await cxgActions.selectCategory(select.metadata, select.values, true);
+        await selectCategory(select.metadata, select.values, true);
       }
     }
-    await utils.clickOn("subset-button");
-    const lassoSelection = await cxgActions.calcDragCoordinates(
+
+    await clickOn("subset-button");
+
+    const lassoSelection = await calcDragCoordinates(
       "layout-graph",
       data.subset.lasso["coordinates-as-percent"]
     );
-    await cxgActions.drag(
-      "layout-graph",
-      lassoSelection.start,
-      lassoSelection.end,
-      true
-    );
-    const cellCount = await cxgActions.cellSet(1);
+
+    await drag("layout-graph", lassoSelection.start, lassoSelection.end, true);
+
+    const cellCount = await getCellSetCount(1);
     expect(cellCount).toBe(data.subset.lasso.count);
   });
+
+  test("undo selection appends the top diff exp genes to user defined genes", async () => {
+    await goToPage(appUrlBase);
+
+    const userDefinedGenes = data.genes.bulkadd;
+    const diffExpGenes = data.diffexp["gene-results"];
+
+    await bulkAddGenes(userDefinedGenes);
+    const userDefinedHistograms = await getAllHistograms(
+      "histogram-user-gene",
+      userDefinedGenes
+    );
+
+    expect(userDefinedHistograms).toEqual(
+      expect.arrayContaining(userDefinedGenes)
+    );
+
+    await subset({ x1: 0.15, y1: 0.1, x2: 0.98, y2: 0.98 });
+    await runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
+
+    const diffExpHistograms = await getAllHistograms(
+      "histogram-diffexp",
+      diffExpGenes
+    );
+
+    expect(diffExpHistograms).toEqual(expect.arrayContaining(diffExpGenes));
+
+    await clickOn("reset-subset-button");
+    const expected = [].concat(userDefinedGenes, diffExpGenes);
+    const userDefinedHistogramsAfterSubset = await getAllHistograms(
+      "histogram-user-gene",
+      expected
+    );
+
+    expect(userDefinedHistogramsAfterSubset).toEqual(
+      expect.arrayContaining(expected)
+    );
+  });
+
+  test("subset selection appends the top diff exp genes to user defined genes", async () => {
+    await goToPage(appUrlBase);
+
+    const userDefinedGenes = data.genes.bulkadd;
+    const diffExpGenes = data.diffexp["gene-results"];
+
+    await bulkAddGenes(userDefinedGenes);
+    const userDefinedHistograms = await getAllHistograms(
+      "histogram-user-gene",
+      userDefinedGenes
+    );
+
+    expect(userDefinedHistograms).toEqual(
+      expect.arrayContaining(userDefinedGenes)
+    );
+
+    await subset({ x1: 0.15, y1: 0.1, x2: 0.98, y2: 0.98 });
+    await runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
+
+    const diffExpHistograms = await getAllHistograms(
+      "histogram-diffexp",
+      diffExpGenes
+    );
+
+    expect(diffExpHistograms).toEqual(expect.arrayContaining(diffExpGenes));
+
+    await subset({ x1: 0.16, y1: 0.11, x2: 0.97, y2: 0.97 });
+
+    const expected = [].concat(userDefinedGenes, diffExpGenes);
+    const userDefinedHistogramsAfterSubset = await getAllHistograms(
+      "histogram-user-gene",
+      expected
+    );
+
+    expect(userDefinedHistogramsAfterSubset).toEqual(
+      expect.arrayContaining(expected)
+    );
+  });
 });
 
 describe("scatter plot", () => {
   test("scatter plot appears", async () => {
-    await cxgActions.reset();
-    const testGenes = data.scatter.genes;
-    await utils.clickOn("section-bulk-add");
-    await utils.typeInto("input-bulk-add", Object.values(testGenes).join(","));
-    await page.keyboard.press("Enter");
-    await utils.clickOn(`plot-x-${data.scatter.genes.x}`);
-    await utils.clickOn(`plot-y-${data.scatter.genes.y}`);
-    await utils.waitByID("scatterplot");
+    await goToPage(appUrlBase);
+
+    await bulkAddGenes(Object.values(data.scatter.genes));
+    await clickOn(`plot-x-${data.scatter.genes.x}`);
+    await clickOn(`plot-y-${data.scatter.genes.y}`);
+    await waitByID("scatterplot");
   });
 });
 
 describe("clipping", () => {
   test("clip continuous", async () => {
-    await cxgActions.clip(data.clip.min, data.clip.max);
+    await goToPage(appUrlBase);
+
+    await clip(data.clip.min, data.clip.max);
     const histBrushableAreaId = `histogram-${data.clip.metadata}-plot-brushable-area`;
-    const coords = await cxgActions.calcDragCoordinates(
+    const coords = await calcDragCoordinates(
       histBrushableAreaId,
       data.clip["coordinates-as-percent"]
     );
-    await cxgActions.drag(histBrushableAreaId, coords.start, coords.end);
-    const cellCount = await cxgActions.cellSet(1);
+    await drag(histBrushableAreaId, coords.start, coords.end);
+    const cellCount = await getCellSetCount(1);
     expect(cellCount).toBe(data.clip.count);
   });
 
   test("clip gene", async () => {
-    await utils.typeInto("gene-search", data.clip.gene);
+    await goToPage(appUrlBase);
+
+    await typeInto("gene-search", data.clip.gene);
     await page.keyboard.press("Enter");
+
     await page.waitForSelector(`[data-testid='histogram-${data.clip.gene}']`);
-    await cxgActions.clip(data.clip.min, data.clip.max);
+
+    await clip(data.clip.min, data.clip.max);
+
     const histBrushableAreaId = `histogram-${data.clip.gene}-plot-brushable-area`;
-    const coords = await cxgActions.calcDragCoordinates(
+
+    const coords = await calcDragCoordinates(
       histBrushableAreaId,
       data.clip["coordinates-as-percent"]
     );
-    await cxgActions.drag(histBrushableAreaId, coords.start, coords.end);
-    const cellCount = await cxgActions.cellSet(1);
+
+    await drag(histBrushableAreaId, coords.start, coords.end);
+
+    const cellCount = await getCellSetCount(1);
+
     expect(cellCount).toBe(data.clip["gene-cell-count"]);
   });
 });
@@ -309,35 +369,186 @@ describe("clipping", () => {
 // interact with UI elements just that they do not break
 describe("ui elements don't error", () => {
   test("color by", async () => {
-    for (const label in data.categorical) {
-      await utils.clickOn(`colorby-${label}`);
-    }
-    for (const label in data.continuous) {
-      await utils.clickOn(`colorby-${label}`);
+    await goToPage(appUrlBase);
+
+    const allLabels = [
+      ...Object.keys(data.categorical),
+      ...Object.keys(data.continuous),
+    ];
+
+    for (const label of allLabels) {
+      await clickOn(`colorby-${label}`);
     }
   });
 
   test("color by for gene", async () => {
-    await utils.typeInto("gene-search", data.genes.search);
+    await goToPage(appUrlBase);
+
+    await typeInto("gene-search", data.genes.search);
     await page.keyboard.press("Enter");
     await page.waitForSelector(
       `[data-testid='histogram-${data.genes.search}']`
     );
-    await utils.clickOn(`colorby-${data.genes.search}`);
+    await clickOn(`colorby-${data.genes.search}`);
   });
 
   test("pan and zoom", async () => {
-    await utils.clickOn("mode-pan-zoom");
-    const panCoords = await cxgActions.calcDragCoordinates(
+    await goToPage(appUrlBase);
+
+    await clickOn("mode-pan-zoom");
+    const panCoords = await calcDragCoordinates(
       "layout-graph",
       data.pan["coordinates-as-percent"]
     );
-    await cxgActions.drag(
-      "layout-graph",
-      panCoords.start,
-      panCoords.end,
-      false
-    );
-    await page.evaluate(`window.scrollBy(0, 1000);`);
+
+    await drag("layout-graph", panCoords.start, panCoords.end, false);
+
+    await page.evaluate("window.scrollBy(0, 1000);");
   });
 });
+
+describe("centroid labels", () => {
+  test("labels are created", async () => {
+    await goToPage(appUrlBase);
+
+    const labels = Object.keys(data.categorical);
+
+    await clickOn(`colorby-${labels[0]}`);
+    await clickOn("centroid-label-toggle");
+
+    // Toggle colorby for each category and check to see if labels are generated
+    for (let i = 0, { length } = labels; i < length; i += 1) {
+      const label = labels[i];
+      // first label is already enabled
+      if (i !== 0) await clickOn(`colorby-${label}`);
+      const generatedLabels = await getAllByClass("centroid-label");
+      // Number of labels generated should be equal to size of the object
+      expect(generatedLabels).toHaveLength(
+        Object.keys(data.categorical[label]).length
+      );
+    }
+  });
+});
+
+describe("graph overlay", () => {
+  test("transform centroids correctly", async () => {
+    await goToPage(appUrlBase);
+
+    const category = Object.keys(data.categorical)[0];
+
+    await clickOn(`colorby-${category}`);
+    await clickOn("centroid-label-toggle");
+    await clickOn("mode-pan-zoom");
+
+    const panCoords = await calcDragCoordinates(
+      "layout-graph",
+      data.pan["coordinates-as-percent"]
+    );
+
+    const categoryValue = Object.keys(data.categorical[category])[0];
+    const initialCoordinates = await getElementCoordinates(
+      `${categoryValue}-centroid-label`
+    );
+
+    await drag("layout-graph", panCoords.start, panCoords.end, false);
+    const terminalCoordinates = await getElementCoordinates(
+      `${categoryValue}-centroid-label`
+    );
+
+    expect(terminalCoordinates[0] - initialCoordinates[0]).toBeCloseTo(
+      panCoords.end.x - panCoords.start.x
+    );
+    expect(terminalCoordinates[1] - initialCoordinates[1]).toBeCloseTo(
+      panCoords.end.y - panCoords.start.y
+    );
+  });
+});
+
+test("pan zoom mode resets lasso selection", async () => {
+  await goToPage(appUrlBase);
+
+  const panzoomLasso = data.features.panzoom.lasso;
+
+  const lassoSelection = await calcDragCoordinates(
+    "layout-graph",
+    panzoomLasso["coordinates-as-percent"]
+  );
+
+  await drag("layout-graph", lassoSelection.start, lassoSelection.end, true);
+  await waitByID("lasso-element", { visible: true });
+
+  const initialCount = await getCellSetCount(1);
+
+  expect(initialCount).toBe(panzoomLasso.count);
+
+  await clickOn("mode-pan-zoom");
+  await clickOn("mode-lasso");
+
+  const modeSwitchCount = await getCellSetCount(1);
+
+  expect(modeSwitchCount).toBe(initialCount);
+});
+
+test("lasso moves after pan", async () => {
+  await goToPage(appUrlBase);
+
+  const panzoomLasso = data.features.panzoom.lasso;
+  const coordinatesAsPercent = panzoomLasso["coordinates-as-percent"];
+
+  const lassoSelection = await calcDragCoordinates(
+    "layout-graph",
+    coordinatesAsPercent
+  );
+
+  await drag("layout-graph", lassoSelection.start, lassoSelection.end, true);
+  await waitByID("lasso-element", { visible: true });
+
+  const initialCount = await getCellSetCount(1);
+
+  expect(initialCount).toBe(panzoomLasso.count);
+
+  await clickOn("mode-pan-zoom");
+
+  const panCoords = await calcDragCoordinates(
+    "layout-graph",
+    coordinatesAsPercent
+  );
+
+  await drag("layout-graph", panCoords.start, panCoords.end, false);
+  await clickOn("mode-lasso");
+
+  const panCount = await getCellSetCount(2);
+
+  expect(panCount).toBe(initialCount);
+});
+
+const describeIfCalledByMakeFileTarget =
+  process.env.CXG_AUTH_TYPE?.toLowerCase() === "test"
+    ? describe
+    : describe.skip;
+
+describeIfCalledByMakeFileTarget("auth buttons", () => {
+  test("login then logout", async () => {
+    await goToPage(appUrlBase);
+    await clickOnUntil("log-in", async () => {
+      await page.waitForNavigation({ waitUntil: "networkidle0" });
+      await waitByID("user-info");
+    });
+    await logout();
+  });
+});
+
+const conditionalDescribe =
+  process.env.TEST_AUTH_INTEGRATION === "true" ? describe : describe.skip;
+
+conditionalDescribe("AuthN Integration", () => {
+  it("logs in", async () => {
+    await login();
+  });
+
+  it("logs out", async () => {
+    await login();
+    await logout();
+  });
+});
+/* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/e2eAnnotations.test.js

@@ -0,0 +1,355 @@
+/*
+Tests included in this file are specific to annotation features
+*/
+import { appUrlBase, DATASET } from "./config";
+import { datasets } from "./data";
+
+import {
+  clickOn,
+  goToPage,
+  waitByClass,
+  waitByID,
+  getTestId,
+  getTestClass,
+  getAllByClass,
+} from "./puppeteerUtils";
+
+import {
+  assertCategoryDoesNotExist,
+  calcDragCoordinates,
+  createCategory,
+  createLabel,
+  deleteCategory,
+  deleteLabel,
+  drag,
+  expandCategory,
+  renameCategory,
+  renameLabel,
+  subset,
+  duplicateCategory,
+} from "./cellxgeneActions";
+
+const data = datasets[DATASET];
+
+const perTestCategoryName = "TEST-CATEGORY";
+const perTestLabelName = "TEST-LABEL";
+
+async function setup(config) {
+  await goToPage(appUrlBase);
+
+  // setup the test fixtures
+  await createCategory(perTestCategoryName);
+  await createLabel(perTestCategoryName, perTestLabelName);
+
+  if (config.withSubset) {
+    await subset({ x1: 0.1, y1: 0.1, x2: 0.8, y2: 0.8 });
+  }
+
+  await waitByClass("autosave-complete");
+}
+
+describe.each([
+  { withSubset: true, tag: "subset" },
+  { withSubset: false, tag: "whole" },
+])("annotations", (config) => {
+  test("create a category", async () => {
+    await setup(config);
+
+    const categoryName = `category-created-${config.tag}`;
+
+    await assertCategoryDoesNotExist(categoryName);
+
+    await createCategory(categoryName);
+
+    await assertCategoryExists(categoryName);
+  });
+
+  test("delete a category", async () => {
+    await setup(config);
+
+    await deleteCategory(perTestCategoryName);
+    await assertCategoryDoesNotExist(perTestCategoryName);
+  });
+
+  test("rename a category", async () => {
+    await setup(config);
+
+    const newCategoryName = `NEW-${config.tag}`;
+
+    await renameCategory(perTestCategoryName, newCategoryName);
+    await assertCategoryDoesNotExist(perTestCategoryName);
+    await assertCategoryExists(newCategoryName);
+  });
+
+  test("create a label", async () => {
+    await setup(config);
+
+    const labelName = `new-label-${config.tag}`;
+
+    await assertLabelDoesNotExist(perTestCategoryName, labelName);
+
+    await createLabel(perTestCategoryName, labelName);
+
+    await assertLabelExists(perTestCategoryName, labelName);
+  });
+
+  test("delete a label", async () => {
+    await setup(config);
+
+    await deleteLabel(perTestCategoryName, perTestLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName, perTestLabelName);
+  });
+
+  test("rename a label", async () => {
+    await setup(config);
+
+    const newLabelName = "my-cool-new-label";
+
+    await assertLabelDoesNotExist(perTestCategoryName, newLabelName);
+    await renameLabel(perTestCategoryName, perTestLabelName, newLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName, perTestLabelName);
+    await assertLabelExists(perTestCategoryName, newLabelName);
+  });
+
+  test("check cell count for a label loaded from file", async () => {
+    await setup(config);
+
+    const duplicateCategoryName = "duplicate";
+    await duplicateCategory(duplicateCategoryName);
+
+    await page.reload({ waitUntil: ["networkidle0", "domcontentloaded"] });
+
+    const firstCategoryExpandIcon = await expect(page).toMatchElement(
+      getTestClass("category-expand")
+    );
+
+    await firstCategoryExpandIcon.click();
+
+    const expectedCategoryRow = await expect(page).toMatchElement(
+      getTestClass("categorical-row")
+    );
+    const expectedLabelName = await getInnerText(
+      expectedCategoryRow,
+      "categorical-value"
+    );
+    const expectedLabelCount = await getInnerText(
+      expectedCategoryRow,
+      "categorical-value-count"
+    );
+
+    await expandCategory(duplicateCategoryName);
+
+    const expectedCategory = await expect(page).toMatchElement(
+      getTestClass("category")
+    );
+
+    const actualCategoryRow = await expect(expectedCategory).toMatchElement(
+      getTestClass("categorical-row")
+    );
+    const actualLabelName = await getInnerText(
+      actualCategoryRow,
+      "categorical-value"
+    );
+    const actualLabelCount = await getInnerText(
+      actualCategoryRow,
+      "categorical-value-count"
+    );
+
+    expect(actualLabelName).toBe(expectedLabelName);
+    expect(actualLabelCount).toBe(expectedLabelCount);
+
+    async function getInnerText(element, className) {
+      return element.$eval(getTestClass(className), (node) => node?.innerText);
+    }
+  });
+
+  test("assign cells to a label", async () => {
+    await setup(config);
+
+    await expandCategory(perTestCategoryName);
+
+    const lassoSelection = await calcDragCoordinates(
+      "layout-graph",
+      data.categoryLabel.lasso["coordinates-as-percent"]
+    );
+
+    await drag("layout-graph", lassoSelection.start, lassoSelection.end, true);
+    await waitByID("lasso-element", { visible: true });
+    await clickOn(`${perTestCategoryName}:${perTestLabelName}:see-actions`);
+    await clickOn(
+      `${perTestCategoryName}:${perTestLabelName}:add-current-selection-to-this-label`
+    );
+
+    const result = await waitByID(
+      `categorical-value-count-${perTestCategoryName}-${perTestLabelName}`
+    );
+
+    expect(await result.evaluate((node) => node.innerText)).toBe(
+      data.categoryLabel.newCount.bySubsetConfig[config.withSubset]
+    );
+  });
+
+  test("undo/redo category creation", async () => {
+    await setup(config);
+
+    const categoryName = `category-created-undo-${config.tag}`;
+
+    await assertCategoryDoesNotExist(categoryName);
+    await createCategory(categoryName);
+    await assertCategoryExists(categoryName);
+    await clickOn("undo");
+    await assertCategoryDoesNotExist(categoryName);
+    await clickOn("redo");
+    await assertCategoryExists(categoryName);
+  });
+
+  test("undo/redo category deletion", async () => {
+    await setup(config);
+
+    const categoryName = `category-deleted-undo-${config.tag}`;
+
+    await createCategory(categoryName);
+    await assertCategoryExists(categoryName);
+    await deleteCategory(categoryName);
+    await assertCategoryDoesNotExist(categoryName);
+    await clickOn("undo");
+    await assertCategoryExists(categoryName);
+    await clickOn("redo");
+    await assertCategoryDoesNotExist(categoryName);
+  });
+
+  test("undo/redo category rename", async () => {
+    await setup(config);
+
+    const newCategoryName = `category-renamed-undo-${config.tag}`;
+
+    await assertCategoryDoesNotExist(newCategoryName);
+    await renameCategory(perTestCategoryName, newCategoryName);
+    await assertCategoryExists(newCategoryName);
+    await assertCategoryDoesNotExist(perTestCategoryName);
+    await clickOn("undo");
+    await assertCategoryExists(perTestCategoryName);
+    await assertCategoryDoesNotExist(newCategoryName);
+    await clickOn("redo");
+    await assertCategoryExists(newCategoryName);
+    await assertCategoryDoesNotExist(perTestCategoryName);
+  });
+
+  test("undo/redo label creation", async () => {
+    await setup(config);
+
+    const labelName = `label-created-undo-${config.tag}`;
+
+    await assertLabelDoesNotExist(perTestCategoryName, labelName);
+    await createLabel(perTestCategoryName, labelName);
+    await assertLabelExists(perTestCategoryName, labelName);
+    await clickOn("undo");
+    await assertLabelDoesNotExist(perTestCategoryName);
+    await clickOn("redo");
+    await assertLabelExists(perTestCategoryName, labelName);
+  });
+
+  test("undo/redo label deletion", async () => {
+    await setup(config);
+
+    await deleteLabel(perTestCategoryName, perTestLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName);
+    await clickOn("undo");
+    await assertLabelExists(perTestCategoryName, perTestLabelName);
+    await clickOn("redo");
+    await assertLabelDoesNotExist(perTestCategoryName);
+  });
+
+  test("undo/redo label rename", async () => {
+    await setup(config);
+
+    const newLabelName = `label-renamed-undo-${config.tag}`;
+
+    await assertLabelDoesNotExist(perTestCategoryName, newLabelName);
+    await renameLabel(perTestCategoryName, perTestLabelName, newLabelName);
+    await assertLabelExists(perTestCategoryName, newLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName, perTestLabelName);
+    await clickOn("undo");
+    await assertLabelExists(perTestCategoryName, perTestLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName, newLabelName);
+    await clickOn("redo");
+    await assertLabelExists(perTestCategoryName, newLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName, perTestLabelName);
+  });
+
+  test("stacked bar graph renders", async () => {
+    await setup(config);
+
+    await expandCategory(perTestCategoryName);
+
+    await clickOn(`colorby-louvain`);
+
+    const labels = await getAllByClass("categorical-row");
+
+    const result = await Promise.all(
+      labels.map((label) => {
+        return page.evaluate((element) => {
+          return element.outerHTML;
+        }, label);
+      })
+    );
+
+    expect(result).toMatchSnapshot();
+  });
+
+  test("truncate midpoint whitespace", async () => {
+    await setup(config);
+    const newLabelName = "123 456";
+    await renameLabel(perTestCategoryName, perTestLabelName, newLabelName);
+    const value = await waitByID(
+      `categorical-value-${perTestCategoryName}-${newLabelName}`
+    );
+    const result = await page.evaluate((elem) => elem.outerHTML, value);
+    expect(result).toMatchSnapshot();
+  });
+
+  test("truncate single character", async () => {
+    await setup(config);
+    const newLabelName = "T";
+    await renameLabel(perTestCategoryName, perTestLabelName, newLabelName);
+    const value = await waitByID(
+      `categorical-value-${perTestCategoryName}-${newLabelName}`
+    );
+    const result = await page.evaluate((elem) => elem.outerHTML, value);
+    expect(result).toMatchSnapshot();
+  });
+
+  async function assertCategoryExists(categoryName) {
+    const handle = await waitByID(`${categoryName}:category-label`);
+
+    const result = await handle.evaluate((node) =>
+      node.getAttribute("aria-label")
+    );
+
+    return expect(result).toBe(categoryName);
+  }
+
+  async function assertLabelExists(categoryName, labelName) {
+    await expect(page).toMatchElement(
+      getTestId(`${categoryName}:category-expand`)
+    );
+
+    await expandCategory(categoryName);
+
+    const previous = await waitByID(
+      `categorical-value-${categoryName}-${labelName}`
+    );
+
+    expect(
+      await previous.evaluate((node) => node.getAttribute("aria-label"))
+    ).toBe(labelName);
+  }
+
+  async function assertLabelDoesNotExist(categoryName, labelName) {
+    await expandCategory(categoryName);
+    const result = await page.$(
+      `[data-testid='categorical-value-${categoryName}-${labelName}']`
+    );
+    expect(result).toBeNull();
+  }
+});

client/__tests__/e2e/e2eJestConfig.json

@@ -1,10 +1,10 @@
 {
+  "testRunner": "jest-circus/runner",
   "preset": "jest-puppeteer",
-  "testMatch": [
-    "**/__tests__/**/?(*.)(spec|test).js?(x)"
-  ],
-  "testURL": "http://localhost/",
-  "setupFiles": [
-    "../setupMissingGlobals.js"
-  ]
+  "testMatch": ["**/__tests__/**/?(*.)(spec|test).js?(x)"],
+  "setupFiles": ["../setupMissingGlobals.js"],
+  "setupFilesAfterEnv": ["expect-puppeteer", "./puppeteer.setup.js"],
+  "globalSetup": "jest-environment-puppeteer/setup",
+  "globalTeardown": "jest-environment-puppeteer/teardown",
+  "testEnvironment": "./screenshot_env.js"
 }

client/__tests__/e2e/feature.test.js

@@ -1,118 +0,0 @@
-/*
-NOT run in Travis CI
-
-UX tests using puppeteer to be run locally.
-
-To run locally, ensure you are running the client is running on port 3000.
-Then run jest --verbose false --config __tests__/e2e/e2eJestConfig.json feature.
- */
-
-import puppeteer from "puppeteer";
-import { appUrlBase, DEBUG, DEV, DATASET } from "./config";
-import { puppeteerUtils, cellxgeneActions } from "./puppeteerUtils";
-import { datasets } from "./data";
-
-let browser, page, utils, cxgActions, spy;
-const browserViewport = { width: 1280, height: 960 };
-let data = datasets[DATASET].features;
-
-if (DEBUG) jest.setTimeout(100000);
-if (DEV) jest.setTimeout(10000);
-
-beforeAll(async () => {
-  const browserParams = DEV
-    ? { headless: false, slowMo: 5 }
-    : DEBUG
-    ? { headless: false, slowMo: 100, devtools: true }
-    : {};
-  browser = await puppeteer.launch(browserParams);
-  page = await browser.newPage();
-  await page.setViewport(browserViewport);
-  if (DEV || DEBUG) {
-    page.on("console", msg => console.log(`PAGE LOG: ${msg.text()}`));
-  }
-  page.on("pageerror", err => {
-    throw new Error(`Console error: ${err}`);
-  });
-  utils = puppeteerUtils(page);
-  cxgActions = cellxgeneActions(page);
-});
-
-beforeEach(async () => {
-  await page.goto(appUrlBase);
-});
-
-afterAll(() => {
-  if (!DEBUG) {
-    browser.close();
-  }
-});
-
-describe("zoom interaction", async () => {
-  // Skip this test since UI is to hide lasso path when switching modes
-  test.skip("lasso visible after switching modes to pan/zoom", async () => {
-    const lassoSelection = await cxgActions.calcDragCoordinates(
-      "layout-graph",
-      data.panzoom.lasso["coordinates-as-percent"]
-    );
-    await cxgActions.drag(
-      "layout-graph",
-      lassoSelection.start,
-      lassoSelection.end,
-      true
-    );
-    await utils.waitByID("lasso-element", { visible: true });
-    await utils.clickOn("mode-pan-zoom");
-    await utils.waitByID("lasso-element", { visible: true });
-  });
-
-  test("pan zoom mode resets lasso selection", async () => {
-    const lassoSelection = await cxgActions.calcDragCoordinates(
-      "layout-graph",
-      data.panzoom.lasso["coordinates-as-percent"]
-    );
-    await cxgActions.drag(
-      "layout-graph",
-      lassoSelection.start,
-      lassoSelection.end,
-      true
-    );
-    await utils.waitByID("lasso-element", { visible: true });
-    const initialCount = await cxgActions.cellSet(1);
-    expect(initialCount).toBe(data.panzoom.lasso.count);
-    await utils.clickOn("mode-pan-zoom");
-    await utils.clickOn("mode-lasso");
-    const modeSwitchCount = await cxgActions.cellSet(1);
-    expect(modeSwitchCount).toBe(initialCount);
-  });
-
-  test("lasso moves after pan", async () => {
-    const lassoSelection = await cxgActions.calcDragCoordinates(
-      "layout-graph",
-      data.panzoom.lasso["coordinates-as-percent"]
-    );
-    await cxgActions.drag(
-      "layout-graph",
-      lassoSelection.start,
-      lassoSelection.end,
-      true
-    );
-    await utils.waitByID("lasso-element", { visible: true });
-    const initialCount = await cxgActions.cellSet(1);
-    expect(initialCount).toBe(data.panzoom.lasso.count);
-    await utils.clickOn("mode-pan-zoom");
-    const panCoords = await cxgActions.calcDragCoordinates(
-      "layout-graph",
-      data.panzoom.lasso["coordinates-as-percent"]
-    );
-    await cxgActions.drag(
-      "layout-graph",
-      panCoords.start,
-      panCoords.end,
-      false
-    );
-    await utils.clickOn("mode-lasso");
-    const panCount = await cxgActions.cellSet(2);
-    expect(panCount).toBe(initialCount);
-  });
-});

client/__tests__/e2e/puppeteer.setup.js

@@ -0,0 +1,58 @@
+/**
+ * `client/jest-puppeteer.config.js` is for configuring Puppeteer's launch config options
+ * `client/__tests__/e2e/puppeteer.setup.js` is for configuring `jest`, `browser`,
+ * and `page` objects
+ */
+
+import { setDefaultOptions } from "expect-puppeteer";
+import { isDebug, isDev } from "./config";
+import * as ENV_DEFAULT from "../../../environment.default.json";
+
+// (thuang): This is the max time a test can take to run.
+// Since when debugging, we run slowMo and !headless, this means
+// a test can take more time to finish, so we don't want
+// jest to shut off the test too soon
+jest.setTimeout(2 * 60 * 1000);
+setDefaultOptions({ timeout: 20 * 1000 });
+
+jest.retryTimes(ENV_DEFAULT.RETRY_ATTEMPTS);
+
+beforeEach(async () => {
+  await jestPuppeteer.resetBrowser();
+
+  const userAgent = await browser.userAgent();
+  await page.setUserAgent(`${userAgent}bot`);
+
+  await page._client.send("Animation.setPlaybackRate", { playbackRate: 12 });
+
+  page.on("pageerror", (err) => {
+    throw new Error(`Console error: ${err}`);
+  });
+
+  page.on("error", (err) => {
+    throw new Error(`Console error: ${err}`);
+  });
+
+  page.on("console", async (msg) => {
+    if (isDev || isDebug) {
+      // If there is a console.error but an error is not thrown, this will ensure the test fails
+      console.log(`PAGE LOG: ${msg.text()}`);
+      if (msg.type() === "error") {
+        // TODO: chromium does not currently support the CSP directive on the
+        // line below, so we swallow this error. Remove this when the test
+        // suite uses a browser version that supports this directive.
+        if (
+          msg.text() ===
+          "Unrecognized Content-Security-Policy directive 'require-trusted-types-for'.\n"
+        ) {
+          return;
+        }
+        const errorMsgText = await Promise.all(
+          // TODO can we do this without internal properties?
+          msg.args().map((arg) => arg._remoteObject.description)
+        );
+        throw new Error(`Console error: ${errorMsgText}`);
+      }
+    }
+  });
+});

client/__tests__/e2e/puppeteerUtils.js

@@ -1,211 +1,131 @@
-export const puppeteerUtils = puppeteerPage => ({
-  async waitByID(testid, props = {}) {
-    return await puppeteerPage.waitForSelector(
-      `[data-testid='${testid}']`,
-      props
-    );
-  },
+/* eslint-disable no-await-in-loop -- await in loop is needed to emulate sequential user actions */
+export function getTestId(id) {
+  return `[data-testid='${id}']`;
+}
 
-  async waitByClass(testclass, props = {}) {
-    return await puppeteerPage.waitForSelector(
-      `[data-testclass='${testclass}']`,
-      props
-    );
-  },
+export function getTestClass(className) {
+  return `[data-testclass='${className}']`;
+}
 
-  async waitForAllByIds(testids, props = {}) {
-    await Promise.all(
-      testids.map(testid =>
-        puppeteerPage.waitForSelector(`[data-testid='${testid}']`)
-      )
-    );
-  },
+export async function waitByID(testId, props = {}) {
+  return page.waitForSelector(getTestId(testId), props);
+}
 
-  async getAllByClass(testclass, props = {}) {
-    const elements = await puppeteerPage.$$eval(
-      `[data-testclass=${testclass}]`,
-      els => {
-        return els.map(el => {
-          return el.dataset.testid;
-        });
-      }
-    );
-    return elements;
-  },
+export async function waitByClass(testClass, props = {}) {
+  await page.waitForSelector(`[data-testclass='${testClass}']`, props);
+}
 
-  async typeInto(testid, text) {
-    // only works for text without special characters
-    await this.waitByID(testid);
-    const selector = `[data-testid='${testid}']`;
-    // type ahead can be annoying if you don't pause before you type
-    await puppeteerPage.click(selector);
-    await puppeteerPage.waitFor(200);
-    await puppeteerPage.type(selector, text);
-  },
+export async function waitForAllByIds(testIds) {
+  await Promise.all(
+    testIds.map((testId) => page.waitForSelector(getTestId(testId)))
+  );
+}
 
-  async clearInputAndTypeInto(testid, text) {
-    await this.waitByID(testid);
-    const selector = `[data-testid='${testid}']`;
-    // only works for text without special characters
-    // type ahead can be annoying if you don't pause before you type
-    await puppeteerPage.click(selector);
-    await puppeteerPage.waitFor(200);
-    // select all
+export async function getAllByClass(testClass) {
+  return page.$$(`[data-testclass=${testClass}]`);
+}
 
-    await puppeteerPage.click(selector, { clickCount: 3 });
-    await puppeteerPage.keyboard.type("Backspace");
-    await puppeteerPage.type(selector, text);
-  },
+export async function typeInto(testId, text) {
+  // blueprint's  typeahead is treating typing weird, clicking & waiting first solves this
+  // only works for text without special characters
+  await waitByID(testId);
+  const selector = getTestId(testId);
+  // type ahead can be annoying if you don't pause before you type
+  await page.click(selector);
+  await page.waitFor(200);
+  await page.type(selector, text);
+}
 
-  async clickOn(testid) {
-    await this.waitByID(testid);
-    await puppeteerPage.click(`[data-testid='${testid}']`);
-    await puppeteerPage.waitFor(50);
-  },
+export async function clearInputAndTypeInto(testId, text) {
+  await waitByID(testId);
+  const selector = getTestId(testId);
+  // only works for text without special characters
+  // type ahead can be annoying if you don't pause before you type
+  await page.click(selector);
+  await page.waitFor(200);
+  // select all
+  await page.click(selector, { clickCount: 3 });
+  await page.keyboard.press("Backspace");
+  await page.type(selector, text);
+}
 
-  async getOneElementInnerHTML(selector) {
-    await puppeteerPage.waitForSelector(selector);
-    let text = await puppeteerPage.$eval(selector, el => el.innerHTML);
-    return text;
-  },
+export async function clickOn(testId, options = {}) {
+  await expect(page).toClick(getTestId(testId), options);
+}
 
-  async getOneElementInnerText(selector) {
-    await puppeteerPage.waitForSelector(selector);
-    let text = await puppeteerPage.$eval(selector, el => el.innerText);
-    return text;
-  }
-});
+/**
+ * (thuang): There are times when Puppeteer clicks on a button and the page doesn't respond.
+ * So I added clickOnUntil() to retry clicking until a given condition is met.
+ */
+export async function clickOnUntil(testId, assert) {
+  const MAX_RETRY = 10;
+  const WAIT_FOR_MS = 200;
 
-export const cellxgeneActions = puppeteerPage => ({
-  async drag(testid, start, end, lasso = false) {
-    const layout = await puppeteerUtils(puppeteerPage).waitByID(testid);
-    const elBox = await layout.boxModel();
-    const x1 = elBox.content[0].x + start.x;
-    const x2 = elBox.content[0].x + end.x;
-    const y1 = elBox.content[0].y + start.y;
-    const y2 = elBox.content[0].y + end.y;
-    await puppeteerPage.mouse.move(x1, y1);
-    await puppeteerPage.mouse.down();
-    if (lasso) {
-      await puppeteerPage.mouse.move(x2, y1);
-      await puppeteerPage.mouse.move(x2, y2);
-      await puppeteerPage.mouse.move(x1, y2);
-      await puppeteerPage.mouse.move(x1, y1);
-    } else {
-      await puppeteerPage.mouse.move(x2, y2);
-    }
-    await puppeteerPage.mouse.up();
-  },
+  let retry = 0;
 
-  async getAllHistograms(testclass, testids) {
-    const histTestIds = testids.map(tid => `histogram-${tid}`);
-    // these load asynchronously, so we need to wait for each histogram individually
-    await puppeteerUtils(puppeteerPage).waitForAllByIds(histTestIds);
-    const allHistograms = await puppeteerUtils(puppeteerPage).getAllByClass(
-      testclass
-    );
-    return allHistograms.map(hist =>
-      hist.substr("histogram_".length, hist.length)
-    );
-  },
-
-  async getAllCategoriesAndCounts(category) {
-    await puppeteerUtils(puppeteerPage).waitByClass("categorical-row");
-    const categories = await puppeteerPage.$$eval(
-      `[data-testid="category-${category}"] [data-testclass='categorical-row']`,
-      els => {
-        let result = {};
-        els.forEach(el => {
-          const cat = el.querySelector("[data-testclass='categorical-value']")
-            .innerText;
-          const count = el.querySelector(
-            "[data-testclass='categorical-value-count']"
-          ).innerText;
-          result[cat] = count;
-        });
-        return result;
-      }
-    );
-    return categories;
-  },
-
-  async cellSet(num) {
-    await puppeteerUtils(puppeteerPage).clickOn(`cellset-button-${num}`);
-    return await puppeteerUtils(puppeteerPage).getOneElementInnerText(
-      `[data-testid='cellset-count-${num}']`
-    );
-  },
-
-  async resetCategory(category) {
-    const checkboxId = `category-select-${category}`;
-    await puppeteerUtils(puppeteerPage).waitByID(checkboxId);
-    const checkedPseudoclass = await puppeteerPage.$eval(
-      `[data-testid='${checkboxId}']`,
-      el => {
-        return el.matches(":checked");
-      }
-    );
-    if (!checkedPseudoclass) {
-      await puppeteerUtils(puppeteerPage).clickOn(checkboxId);
-    }
+  while (retry < MAX_RETRY) {
     try {
-      const categoryRow = await puppeteerUtils(puppeteerPage).waitByID(
-        `category-expand-${category}`
-      );
-      const isExpanded = await categoryRow.$(
-        "[data-testclass='category-expand-is-expanded']"
-      );
-      if (isExpanded) {
-        await puppeteerUtils(puppeteerPage).clickOn(
-          `category-expand-${category}`
-        );
-      }
-    } catch {}
-  },
+      await clickOn(testId);
+      await assert();
 
-  async calcDragCoordinates(testid, coordinateAsPercent) {
-    const el = await puppeteerUtils(puppeteerPage).waitByID(testid);
-    const size = await el.boxModel();
-    const coords = {
-      start: {
-        x: Math.floor(size.width * coordinateAsPercent.x1),
-        y: Math.floor(size.height * coordinateAsPercent.y1)
-      },
-      end: {
-        x: Math.floor(size.width * coordinateAsPercent.x2),
-        y: Math.floor(size.height * coordinateAsPercent.y2)
-      }
-    };
-    return coords;
-  },
+      break;
+    } catch (error) {
+      retry += 1;
 
-  async selectCategory(category, values, reset = true) {
-    if (reset) await this.resetCategory(category);
-    await puppeteerUtils(puppeteerPage).clickOn(`category-expand-${category}`);
-    await puppeteerUtils(puppeteerPage).clickOn(`category-select-${category}`);
-    for (const val of values) {
-      await puppeteerUtils(puppeteerPage).clickOn(
-        `categorical-value-select-${category}-${val}`
-      );
+      await page.waitFor(WAIT_FOR_MS);
     }
-  },
-
-  async reset() {
-    await puppeteerUtils(puppeteerPage).clickOn("reset");
-    // loading state never actually happens, reset is too fast
-    await page.waitFor(200);
-  },
-
-  async clip(min = 0, max = 100) {
-    await puppeteerUtils(puppeteerPage).clickOn("visualization-settings");
-    await puppeteerUtils(puppeteerPage).clearInputAndTypeInto(
-      "clip-min-input",
-      min
-    );
-    await puppeteerUtils(puppeteerPage).clearInputAndTypeInto(
-      "clip-max-input",
-      max
-    );
-    await puppeteerUtils(puppeteerPage).clickOn("clip-commit");
   }
-});
+
+  if (retry === MAX_RETRY) {
+    throw Error("clickOnUntil() assertion failed!");
+  }
+}
+
+export async function getOneElementInnerHTML(selector, options = {}) {
+  await page.waitForSelector(selector, options);
+
+  return page.$eval(selector, (el) => el.innerHTML);
+}
+
+export async function getOneElementInnerText(selector) {
+  expect(page).toMatchElement(selector);
+
+  return page.$eval(selector, (el) => el.innerText);
+}
+
+export async function getElementCoordinates(testId) {
+  return page.$eval(getTestId(testId), (elem) => {
+    const { left, top } = elem.getBoundingClientRect();
+    return [left, top];
+  });
+}
+
+async function clickTermsOfService() {
+  if (!(await isElementPresent(getTestId("tos-cookies-accept")))) return;
+
+  await clickOn("tos-cookies-accept");
+}
+
+async function nameNewAnnotation() {
+  if (await isElementPresent(getTestId("annotation-dialog"))) {
+    await typeInto("new-annotation-name", "ignoreE2E");
+    await clickOn("submit-annotation");
+
+    // wait for the page to load
+    await waitByClass("autosave-complete");
+  }
+}
+
+export async function goToPage(url) {
+  await page.goto(url, {
+    waitUntil: "networkidle0",
+  });
+
+  await nameNewAnnotation();
+  await clickTermsOfService();
+}
+
+export async function isElementPresent(selector, options) {
+  return Boolean(await page.$(selector, options));
+}
+/* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/screenshot_env.js

@@ -0,0 +1,36 @@
+const PuppeteerEnvironment = require("jest-environment-puppeteer");
+require("jest-circus");
+const ENV_DEFAULT = require("../../../environment.default.json");
+
+const takeScreenshot = require("./takeScreenshot");
+
+class ScreenshotEnvironment extends PuppeteerEnvironment {
+  async handleTestEvent(event, state) {
+    if (["test_start", "test_done"].includes(event.name)) {
+      console.log("------------------event name:\n", event.name);
+      console.log("~~~~ Current test errors\n", new Date(), event.test.errors);
+      console.log("~~~~ Current test\n", new Date(), event.test);
+    }
+
+    if (event.name === "error") {
+      console.log("error event:", JSON.stringify(event));
+    }
+
+    if (event.name === "test_fn_failure" || event.name === "hook_failure") {
+      console.log("------------------event name:\n", event.name);
+      console.log(">>>> Current state\n", new Date(), state);
+      console.log("===> Failure event\n", new Date(), event);
+
+      // (thuang): We only want to take screenshot on the last try
+      if (
+        state.currentlyRunningTest.invocations <= ENV_DEFAULT.RETRY_ATTEMPTS
+      ) {
+        return;
+      }
+
+      await takeScreenshot(state.currentlyRunningTest.name, this.global.page);
+    }
+  }
+}
+
+module.exports = ScreenshotEnvironment;

client/__tests__/e2e/takeScreenshot.js

@@ -0,0 +1,17 @@
+function toFilename(name) {
+  return name.replace(/[^a-z0-9.-]+/gi, "-");
+}
+
+async function takeScreenshot(currentTestName, page) {
+  const testName = toFilename(currentTestName);
+
+  // Take a screenshot at the point of failure
+  const date = new Date().toISOString();
+  const screenshotName = `${date}-${testName}.png`;
+
+  await page.screenshot({
+    path: `./__tests__/screenshots/ignoreE2E-screenshot-${screenshotName}`,
+  });
+}
+
+module.exports = takeScreenshot;

client/__tests__/e2e/test_config.yaml

@@ -0,0 +1,47 @@
+server:
+  app:
+    force_https: true
+
+    # By default, cellxgene will serve api requests from the same base url as the webpage.
+    # In general api_base_url and web_base_url will not need to be set.
+    # There are two reasons to set these parameters:
+    #  1. Oauth authentication is used; the oauth server will redirect back to the api_base_url after login,
+    #     which then redirects back to the web_base_url.  If the web_base_url is not set, it will default to
+    #     the api_base_url.  If oauth authentication is used, the api_base_url must be set.
+    #     For a local test (where the server runs on "http://localhost:<port>"), then the api_base_url may be
+    #     set to the string "local".
+    #  2. The cellxgene deploymnent is in an environment where the webpage and api have
+    #     different base urls.  In this case both api_base_url and web_base_url must be set.
+    #     It is up to the server admin to ensure that the networking is setup correctly for this environment.
+    api_base_url: http://localhost:5005
+    web_base_url: http://localhost:3000
+
+  authentication:
+    # The authentication types may be "none", "session", "oauth"
+    # none:  No authentication support, features like user_annotations must not be enabled.
+    # session:  A session based userid is automatically generated. (no params needed)
+    # oauth: oauth2 is used for authentication;  parameters are defined in params_oauth.
+    type: test
+
+dataset:
+  app:
+    about_legal_tos: null
+    about_legal_privacy: null
+
+  presentation:
+    max_categories: 1000
+    custom_colors: true
+
+  user_annotations:
+    enable: false
+    type: local_file_csv
+    local_file_csv:
+      directory: null
+      file: null
+    ontology:
+      enable: false
+      obo_location: null
+
+  embeddings:
+    names: []
+    enable_reembedding: false

client/__tests__/reducers/cascade.test.js

@@ -26,7 +26,7 @@ describe("cascade", () => {
           expect(nextSharedState).toStrictEqual({});
           expect(prevSharedState).toBe(topLevelState);
           return 0;
-        }
+        },
       ],
       [
         "bar",
@@ -36,8 +36,8 @@ describe("cascade", () => {
           expect(nextSharedState).toStrictEqual({ foo: 0 });
           expect(prevSharedState).toBe(topLevelState);
           return 99;
-        }
-      ]
+        },
+      ],
     ]);
 
     const nextState = reducer(topLevelState, topLevelAction);

client/__tests__/reducers/undoable.test.js

@@ -23,7 +23,7 @@ describe("create", () => {
 describe("undo", () => {
   test("expected state modifications", () => {
     const initialState = { a: 0, b: 1000 };
-    const reducer = state => {
+    const reducer = (state) => {
       return { a: state.a + 1, b: state.b + 1 };
     };
     const undoableReducer = undoable(reducer, ["a"]);
@@ -43,7 +43,7 @@ describe("undo", () => {
 
 describe("redo", () => {
   const initialState = { a: 0, b: 1000 };
-  const reducer = state => {
+  const reducer = (state) => {
     return { a: state.a + 1, b: state.b + 1 };
   };
   let UR;
@@ -58,7 +58,7 @@ describe("redo", () => {
 
     // verify undo->redo reverts state.
     const s2 = UR(UR(s1, { type: "@@undoable/undo" }), {
-      type: "@@undoable/redo"
+      type: "@@undoable/redo",
     });
     expect(s2).toMatchObject({ a: 1, b: 1001 });
 

client/__tests__/screenshots/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

client/__tests__/util/actionHelpers.test.js

@@ -13,16 +13,16 @@ describe("rangeEncodeIndices", () => {
     expect(rangeEncodeIndices([1, 9, 432], 10, true)).toMatchObject([
       1,
       9,
-      432
+      432,
     ]);
     expect(rangeEncodeIndices([1, 9, 432], 10, false)).toMatchObject([
       1,
       9,
-      432
+      432,
     ]);
-    expect(rangeEncodeIndices([0, 1, 2, 3, 9, 10, 432], 2, true)).toMatchObject(
-      [[0, 3], [9, 10], 432]
-    );
+    expect(
+      rangeEncodeIndices([0, 1, 2, 3, 9, 10, 432], 2, true)
+    ).toMatchObject([[0, 3], [9, 10], 432]);
     expect(
       rangeEncodeIndices([0, 1, 2, 3, 9, 10, 432], 2, false)
     ).toMatchObject([[0, 3], [9, 10], 432]);
@@ -34,7 +34,10 @@ describe("rangeEncodeIndices", () => {
     ).toMatchObject([[3, 7], [10, 12], 99]);
     expect(
       rangeEncodeIndices([3, 4, 5, 6, 7, 10, 11, 12], 3, false)
-    ).toMatchObject([[3, 7], [10, 12]]);
+    ).toMatchObject([
+      [3, 7],
+      [10, 12],
+    ]);
     expect(
       rangeEncodeIndices([0, 3, 4, 5, 6, 7, 10, 11, 12], 3, false)
     ).toMatchObject([0, [3, 7], [10, 12]]);

client/__tests__/util/annoMatrix/annoMatrix.test.js

@@ -0,0 +1,316 @@
+// these TWO statements MUST be first in the file, before any other imports
+import { enableFetchMocks } from "jest-fetch-mock";
+import * as serverMocks from "./serverMocks";
+// OK, continue on!
+
+import {
+  AnnoMatrixLoader,
+  clip,
+  isubset,
+  isubsetMask,
+} from "../../../src/annoMatrix";
+import { Dataframe } from "../../../src/util/dataframe";
+
+enableFetchMocks();
+
+describe("AnnoMatrix", () => {
+  let annoMatrix;
+
+  beforeEach(async () => {
+    fetch.resetMocks(); // reset all fetch mocking state
+    annoMatrix = new AnnoMatrixLoader(
+      serverMocks.baseDataURL,
+      serverMocks.schema.schema
+    );
+  });
+
+  describe("basics", () => {
+    test("annomatrix static checks", () => {
+      expect(annoMatrix).toBeDefined();
+      expect(annoMatrix.schema).toMatchObject(serverMocks.schema.schema);
+      expect(annoMatrix.nObs).toEqual(serverMocks.schema.schema.dataframe.nObs);
+      expect(annoMatrix.nVar).toEqual(serverMocks.schema.schema.dataframe.nVar);
+      expect(annoMatrix.isView).toBeFalsy();
+      expect(annoMatrix.viewOf).toBeUndefined();
+      expect(annoMatrix.rowIndex).toBeDefined();
+    });
+
+    test("simple single column fetch", async () => {
+      fetch.once(serverMocks.annotationsObs(["name_0"]));
+
+      const df = await annoMatrix.fetch("obs", "name_0");
+      expect(df).toBeInstanceOf(Dataframe);
+      expect(df.colIndex.labels()).toEqual(["name_0"]);
+      expect(df.dims).toEqual([annoMatrix.nObs, 1]);
+    });
+
+    test("simple multi column fetch", async () => {
+      fetch
+        .once(serverMocks.annotationsObs(["name_0"]))
+        .once(serverMocks.annotationsObs(["n_genes"]));
+
+      await expect(
+        annoMatrix.fetch("obs", ["name_0", "n_genes"])
+      ).resolves.toBeInstanceOf(Dataframe);
+    });
+
+    describe("fetch from field", () => {
+      const getLastTwo = async (field) => {
+        const columnNames = annoMatrix.getMatrixColumns(field).slice(-2);
+        fetch.mockResponses(...columnNames.map(() => serverMocks.responder));
+        await expect(
+          annoMatrix.fetch(field, columnNames)
+        ).resolves.toBeInstanceOf(Dataframe);
+      };
+
+      test("obs", async () => getLastTwo("obs"));
+      test("var", async () => getLastTwo("var"));
+      test("emb", async () => getLastTwo("emb"));
+    });
+
+    test("fetch - test all query forms", async () => {
+      // single string is a column name
+      fetch.once(serverMocks.annotationsObs(["n_genes"]));
+      await expect(annoMatrix.fetch("obs", "n_genes")).resolves.toBeInstanceOf(
+        Dataframe
+      );
+
+      // array of column names, expecting n_genes to be cached.
+      fetch.once(serverMocks.annotationsObs(["percent_mito"]));
+      await expect(
+        annoMatrix.fetch("obs", ["n_genes", "percent_mito"])
+      ).resolves.toBeInstanceOf(Dataframe);
+
+      // more complex value filter query, enumerated
+      fetch.once(serverMocks.responder);
+      await expect(
+        annoMatrix.fetch("X", {
+          field: "var",
+          column: annoMatrix.schema.annotations.var.index,
+          value: "TYMP",
+        })
+      ).resolves.toBeInstanceOf(Dataframe);
+
+      // more complex value filter query, range
+      const varIndex = annoMatrix.schema.annotations.var.index;
+      fetch
+        .once(
+          serverMocks.withExpected("/data/var", [[`var:${varIndex}`, "SUMO3"]])
+        )
+        .once(
+          serverMocks.withExpected("/data/var", [[`var:${varIndex}`, "TYMP"]])
+        );
+      await expect(
+        annoMatrix.fetch("X", [
+          {
+            field: "var",
+            column: varIndex,
+            value: "SUMO3",
+          },
+          {
+            field: "var",
+            column: varIndex,
+            value: "TYMP",
+          },
+        ])
+      ).resolves.toBeInstanceOf(Dataframe);
+      // XXX inspect the wherecache?
+    });
+
+    test("push and pop views", async () => {
+      const am1 = clip(annoMatrix, 0.1, 0.9);
+      expect(am1.viewOf).toBe(annoMatrix);
+      expect(am1.nObs).toEqual(annoMatrix.nObs);
+      expect(am1.nVar).toEqual(annoMatrix.nVar);
+      expect(am1.rowIndex).toBe(annoMatrix.rowIndex);
+
+      const am2 = clip(annoMatrix, 0.1, 0.9);
+      expect(am2.viewOf).toBe(annoMatrix);
+      expect(am2).not.toBe(am1);
+      expect(am2.rowIndex).toBe(annoMatrix.rowIndex);
+    });
+
+    test("schema accessors", () => {
+      expect(annoMatrix.getMatrixFields()).toEqual(
+        expect.arrayContaining(["X", "obs", "emb", "var"])
+      );
+      expect(annoMatrix.getMatrixColumns("obs")).toEqual(
+        expect.arrayContaining(["name_0", "n_genes", "louvain"])
+      );
+      expect(annoMatrix.getColumnSchema("emb", "umap")).toEqual({
+        name: "umap",
+        dims: ["umap_0", "umap_1"],
+        type: "float32",
+      });
+      expect(annoMatrix.getColumnDimensions("emb", "umap")).toEqual([
+        "umap_0",
+        "umap_1",
+      ]);
+    });
+
+    /*
+  	test the mask & label access to subset via isubset and isubsetMask
+  	*/
+    test("isubset", async () => {
+      const rowList = [0, 10];
+      const rowMask = new Uint8Array(annoMatrix.nObs);
+      for (let i = 0; i < rowList.length; i += 1) {
+        rowMask[rowList[i]] = 1;
+      }
+
+      const am1 = isubset(annoMatrix, rowList);
+      const am2 = isubsetMask(annoMatrix, rowMask);
+      expect(am1).not.toBe(am2);
+      expect(am1.nObs).toEqual(2);
+      expect(am1.nObs).toEqual(am2.nObs);
+      expect(am1.nVar).toEqual(am2.nVar);
+
+      fetch
+        .once(serverMocks.annotationsObs(["n_genes"]))
+        .once(serverMocks.annotationsObs(["n_genes"]));
+      const ng1 = await am1.fetch("obs", "n_genes");
+      const ng2 = await am2.fetch("obs", "n_genes");
+      expect(ng1).toHaveLength(ng2.length);
+      expect(ng1.colIndex.labels()).toEqual(ng2.colIndex.labels());
+      expect(ng1.col("n_genes").asArray()).toEqual(
+        ng2.col("n_genes").asArray()
+      );
+    });
+  });
+
+  describe("add/drop column", () => {
+    async function addDrop(base) {
+      expect(base.getMatrixColumns("obs")).not.toContain("foo");
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(base.fetch("obs", "foo")).rejects.toThrow(
+        "unknown column name"
+      );
+
+      /* add */
+      const am1 = base.addObsColumn(
+        { name: "foo", type: "float32", writable: true },
+        Float32Array,
+        0
+      );
+      expect(base.getMatrixColumns("obs")).not.toContain("foo");
+      expect(am1.getMatrixColumns("obs")).toContain("foo");
+      const foo = await am1.fetch("obs", "foo");
+      expect(foo).toBeDefined();
+      expect(foo).toBeInstanceOf(Dataframe);
+      expect(foo).toHaveLength(am1.nObs);
+      expect(foo.col("foo").asArray()).toEqual(
+        new Float32Array(am1.nObs).fill(0)
+      );
+
+      /* drop */
+      const am2 = am1.dropObsColumn("foo");
+      expect(base.getMatrixColumns("obs")).not.toContain("foo");
+      expect(am2.getMatrixColumns("obs")).not.toContain("foo");
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(am2.fetch("obs", "foo")).rejects.toThrow(
+        "unknown column name"
+      );
+    }
+
+    test("add/drop column, without view", async () => {
+      await addDrop(annoMatrix);
+    });
+
+    test("add/drop column, with view", async () => {
+      const am1 = clip(annoMatrix, 0.1, 0.9);
+      await addDrop(am1);
+
+      const am2 = isubset(am1, [0, 1, 2, 20, 30, 400]);
+      await addDrop(am2);
+
+      const am3 = isubset(annoMatrix, [10, 0, 7, 3]);
+      await addDrop(am3);
+
+      const am4 = clip(am3, 0, 1);
+      await addDrop(am4);
+
+      fetch.mockResponse(serverMocks.responder);
+
+      await am1.fetch("obs", am1.getMatrixColumns("obs"));
+      await am2.fetch("obs", am2.getMatrixColumns("obs"));
+      await am3.fetch("obs", am3.getMatrixColumns("obs"));
+      await am4.fetch("obs", am4.getMatrixColumns("obs"));
+
+      fetch.resetMocks();
+
+      await addDrop(am1);
+      await addDrop(am2);
+      await addDrop(am3);
+      await addDrop(am4);
+    });
+  });
+
+  describe("setObsColumnValues", () => {
+    async function addSetDrop(base) {
+      /* add column */
+      let am = base.addObsColumn(
+        {
+          name: "test",
+          type: "categorical",
+          categories: ["unassigned", "red", "green"],
+          writable: true,
+        },
+        Array,
+        "unassigned"
+      );
+
+      const testVal = await am.fetch("obs", "test");
+      expect(testVal.col("test").asArray()).toEqual(
+        new Array(am.nObs).fill("unassigned")
+      );
+
+      /* set values in column */
+      const whichRows = [1, 2, 10];
+      const am1 = await am.setObsColumnValues("test", whichRows, "yo");
+      const testVal1 = await am1.fetch("obs", "test");
+      const expt = new Array(am1.nObs).fill("unassigned");
+      for (let i = 0; i < whichRows.length; i += 1) {
+        const offset = am1.rowIndex.getOffset(whichRows[i]);
+        expt[offset] = "yo";
+      }
+      expect(testVal1).not.toBe(testVal);
+      expect(testVal1.col("test").asArray()).toEqual(expt);
+      expect(am1.getColumnSchema("obs", "test").type).toBe("categorical");
+      expect(am1.getColumnSchema("obs", "test").categories).toEqual(
+        expect.arrayContaining(["unassigned", "red", "green", "yo"])
+      );
+
+      /* drop column */
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      am = am1.dropObsColumn("test");
+      await expect(am.fetch("obs", "test")).rejects.toThrow(
+        "unknown column name"
+      );
+    }
+
+    test("set, without a view", async () => {
+      await addSetDrop(annoMatrix);
+    });
+
+    test("set, with a view", async () => {
+      const am1 = clip(annoMatrix, 0.1, 0.9);
+      await addSetDrop(am1);
+
+      const am2 = isubset(am1, [0, 1, 2, 10, 20, 30, 400]);
+      await addSetDrop(am2);
+
+      const am3 = isubset(annoMatrix, [10, 1, 0, 30, 2]);
+      await addSetDrop(am3);
+
+      fetch.mockResponse(serverMocks.responder);
+
+      await am1.fetch("obs", am1.getMatrixColumns("obs"));
+      await am2.fetch("obs", am2.getMatrixColumns("obs"));
+      await am3.fetch("obs", am3.getMatrixColumns("obs"));
+
+      await addSetDrop(am1);
+      await addSetDrop(am2);
+      await addSetDrop(am3);
+    });
+  });
+});

client/__tests__/util/annoMatrix/crossfilter.test.js

@@ -0,0 +1,688 @@
+// these TWO statements MUST be first in the file, before any other imports
+import { enableFetchMocks } from "jest-fetch-mock";
+import * as serverMocks from "./serverMocks";
+// OK, continue on!
+
+import obsLouvain from "./louvain.json";
+import obsNGenes from "./n_genes.json";
+import embUmap from "./umap.json";
+
+import {
+  AnnoMatrixLoader,
+  AnnoMatrixObsCrossfilter,
+  isubsetMask,
+} from "../../../src/annoMatrix";
+import { rangeFill } from "../../../src/util/range";
+
+enableFetchMocks();
+
+describe("AnnoMatrixCrossfilter", () => {
+  let annoMatrix;
+  let crossfilter;
+
+  beforeEach(async () => {
+    fetch.resetMocks(); // reset all fetch mocking state
+    annoMatrix = new AnnoMatrixLoader(
+      serverMocks.baseDataURL,
+      serverMocks.schema.schema
+    );
+    crossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
+  });
+
+  test("initial state of crossfilter", () => {
+    const { nObs } = annoMatrix;
+
+    expect(crossfilter).toBeDefined();
+    expect(crossfilter.size()).toEqual(nObs);
+    expect(crossfilter.annoMatrix).toBe(annoMatrix);
+
+    /* by default, everything should be selected, even if no data in cache */
+    expect(crossfilter.countSelected()).toEqual(nObs);
+    expect(crossfilter.allSelectedLabels()).toEqual(
+      rangeFill(new Int32Array(nObs))
+    );
+    expect(crossfilter.allSelectedMask()).toEqual(new Uint8Array(nObs).fill(1));
+    expect(crossfilter.fillByIsSelected(new Uint8Array(nObs), 2, 1)).toEqual(
+      new Uint8Array(nObs).fill(2)
+    );
+  });
+
+  describe("select", () => {
+    /*
+    test the selection state via crossfilter proxy
+    */
+
+    test("select loads index", async () => {
+      /*
+      Select should transparently load/create dimension index.
+
+      Internal dimension names are field/col:col:col..., eg,
+
+        obs:louvain
+        emb:umap_0:umap_1
+
+      */
+      expect(crossfilter.obsCrossfilter.dimensionNames()).toEqual([]);
+      expect(
+        crossfilter.obsCrossfilter.hasDimension("obs/louvain")
+      ).toBeFalsy();
+
+      fetch.once(serverMocks.dataframeResponse(["louvain"], [obsLouvain]));
+      let newCrossfilter = await crossfilter.select("obs", "louvain", {
+        mode: "none",
+      });
+
+      expect(newCrossfilter.countSelected()).toEqual(0);
+      expect(
+        newCrossfilter.obsCrossfilter.hasDimension("obs/louvain")
+      ).toBeTruthy();
+      expect(fetch.mock.calls).toHaveLength(1);
+
+      newCrossfilter = await crossfilter.select("obs", "louvain", {
+        mode: "all",
+      });
+      expect(newCrossfilter.countSelected()).toEqual(annoMatrix.nObs);
+    });
+
+    test("simple column select", async () => {
+      let xfltr;
+
+      fetch.once(serverMocks.dataframeResponse(["louvain"], [obsLouvain]));
+      xfltr = await crossfilter.select("obs", "louvain", {
+        mode: "exact",
+        values: ["NK cells", "B cells"],
+      });
+
+      expect(xfltr).toBeDefined();
+      expect(xfltr.countSelected()).toEqual(496);
+      expect(xfltr.allSelectedMask()).toEqual(
+        Uint8Array.from(
+          obsLouvain.map((val) =>
+            val === "NK cells" || val === "B cells" ? 1 : 0
+          )
+        )
+      );
+      expect(xfltr.allSelectedLabels()).toEqual(
+        Int32Array.from(
+          obsLouvain.reduce((acc, val, idx) => {
+            if (val === "NK cells" || val === "B cells") acc.push(idx);
+            return acc;
+          }, [])
+        )
+      );
+      expect(
+        xfltr.fillByIsSelected(new Uint8Array(annoMatrix.nObs), 3, 1)
+      ).toEqual(
+        Uint8Array.from(
+          obsLouvain.map((val) =>
+            val === "NK cells" || val === "B cells" ? 3 : 1
+          )
+        )
+      );
+
+      const df = await annoMatrix.fetch("obs", "louvain");
+      const values = df.col("louvain").asArray();
+      const selected = xfltr.allSelectedMask();
+      values.every(
+        (val, idx) => !["NK cells", "B cells"].includes(val) !== !selected[idx]
+      );
+
+      fetch.once(
+        serverMocks.dataframeResponse(["n_genes"], [new Int32Array(obsNGenes)])
+      );
+      xfltr = await xfltr.select("obs", "n_genes", {
+        mode: "range",
+        lo: 0,
+        hi: 500,
+        inclusive: false,
+      });
+      expect(xfltr.countSelected()).toEqual(33);
+      expect(xfltr.allSelectedLabels()).toEqual(
+        Int32Array.from(
+          obsNGenes.reduce((acc, val, idx) => {
+            const louvain = obsLouvain[idx];
+            if (
+              val >= 0 &&
+              val < 500 &&
+              (louvain === "NK cells" || louvain === "B cells")
+            )
+              acc.push(idx);
+            return acc;
+          }, [])
+        )
+      );
+
+      xfltr = await xfltr.selectAll();
+      expect(xfltr.countSelected()).toEqual(annoMatrix.nObs);
+    });
+
+    test("join column select", async () => {
+      const varIndex = annoMatrix.schema.annotations.var.index;
+
+      const { nObs } = annoMatrix.schema.dataframe;
+      fetch.once(
+        serverMocks.dataframeResponse(
+          ["TEST"],
+          [rangeFill(new Float32Array(nObs), 0, 0.1)]
+        )
+      );
+
+      const xfltr = await crossfilter.select(
+        "X",
+        {
+          field: "var",
+          column: varIndex,
+          value: "TYMP",
+        },
+        {
+          mode: "range",
+          lo: 0,
+          hi: 50,
+          inclusive: true,
+        }
+      );
+
+      expect(xfltr).toBeDefined();
+      expect(xfltr.countSelected()).toEqual(501);
+
+      const df = await annoMatrix.fetch("X", {
+        field: "var",
+        column: varIndex,
+        value: "TYMP",
+      });
+      const values = df.icol(0).asArray();
+      const selected = xfltr.allSelectedMask();
+      values.every((val, idx) => !(val >= 0 && val <= 50) !== !selected[idx]);
+      expect(selected.reduce((acc, val) => (val ? acc + 1 : acc), 0)).toEqual(
+        xfltr.countSelected()
+      );
+    });
+
+    test("spatial column select", async () => {
+      fetch.once(
+        serverMocks.dataframeResponse(
+          ["umap_0", "umap_1"],
+          [Float32Array.from(embUmap[0]), Float32Array.from(embUmap[1])]
+        )
+      );
+      const xfltr = await crossfilter.select("emb", "umap", {
+        mode: "within-rect",
+        minX: 0,
+        minY: 0,
+        maxX: 0.5,
+        maxY: 0.5,
+      });
+      expect(xfltr.countSelected()).toEqual(16);
+    });
+
+    test("select on subset", async () => {
+      const mask = new Uint8Array(annoMatrix.nObs).fill(0);
+      for (let i = 0; i < mask.length; i += 2) {
+        mask[i] = true;
+      }
+      const annoMatrixSubset = isubsetMask(annoMatrix, mask);
+      expect(annoMatrixSubset.nObs).toEqual(Math.floor(annoMatrix.nObs / 2));
+
+      let xfltr = new AnnoMatrixObsCrossfilter(annoMatrixSubset);
+      expect(xfltr.countSelected()).toEqual(annoMatrixSubset.nObs);
+
+      fetch.once(serverMocks.dataframeResponse(["louvain"], [obsLouvain]));
+      xfltr = await xfltr.select("obs", "louvain", {
+        mode: "exact",
+        values: ["NK cells", "B cells"],
+      });
+
+      expect(xfltr).toBeDefined();
+      expect(xfltr.countSelected()).toEqual(240);
+
+      const df = await annoMatrixSubset.fetch("obs", "louvain");
+      const values = df.col("louvain").asArray();
+      const selected = xfltr.allSelectedMask();
+      values.every(
+        (val, idx) => !["NK cells", "B cells"].includes(val) !== !selected[idx]
+      );
+    });
+
+    test("select catches errors", async () => {
+      await expect(crossfilter.select("NADA", "foo")).rejects.toThrow(
+        "Unknown field name"
+      );
+
+      await expect(crossfilter.select("var", "foo")).rejects.toThrow(
+        "unable to obsSelect upon the var dimension"
+      );
+
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(crossfilter.select("obs", "foo")).rejects.toThrow(
+        "unknown column name"
+      );
+    });
+  });
+
+  describe("mutate matrix", () => {
+    /*
+    test the matrix mutators via crossfilter proxy
+    */
+    async function helperAddTestCol(cf, colName, colSchema = null) {
+      expect(
+        cf.annoMatrix.getMatrixColumns("obs").includes(colName)
+      ).toBeFalsy();
+
+      if (colSchema === null) {
+        colSchema = {
+          name: colName,
+          type: "categorical",
+          categories: ["toasty"],
+        };
+      }
+      colSchema.name = colName;
+      const initValue = colSchema.categories[0];
+      const xfltr = cf.addObsColumn(colSchema, Array, initValue);
+      expect(
+        xfltr.annoMatrix.schema.annotations.obs.columns.filter(
+          (v) => v.name === colName
+        )
+      ).toHaveLength(1);
+      const df = await xfltr.annoMatrix.fetch("obs", colName);
+      expect(df.hasCol(colName)).toBeTruthy();
+      return xfltr;
+    }
+
+    test("addObsColumn", async () => {
+      expect(crossfilter.countSelected()).toBe(annoMatrix.nObs);
+      expect(
+        crossfilter.annoMatrix.getMatrixColumns("obs").includes("foo")
+      ).toBeFalsy();
+      const xfltr = crossfilter.addObsColumn(
+        { name: "foo", type: "categorical", categories: ["A"] },
+        Array,
+        "A"
+      );
+
+      // check schema updates correctly.
+      expect(xfltr.countSelected()).toBe(annoMatrix.nObs);
+      expect(
+        xfltr.annoMatrix.getMatrixColumns("obs").includes("foo")
+      ).toBeTruthy();
+      expect(xfltr.annoMatrix.schema.annotations.obsByName.foo).toMatchObject({
+        name: "foo",
+        type: "categorical",
+      });
+      expect(
+        xfltr.annoMatrix.schema.annotations.obs.columns.filter(
+          (v) => v.name === "foo"
+        )
+      ).toHaveLength(1);
+
+      // check data update.
+      const df = await xfltr.annoMatrix.fetch("obs", "foo");
+      expect(
+        df
+          .col("foo")
+          .asArray()
+          .every((v) => v === "A")
+      ).toBeTruthy();
+
+      // check that we catch dups
+      expect(() =>
+        xfltr.addObsColumn(
+          { name: "foo", type: "categorical" },
+          Array,
+          "toasty"
+        )
+      ).toThrow("column already exists");
+      expect(() =>
+        xfltr.addObsColumn(
+          { name: "louvain", type: "categorical" },
+          Array,
+          "toasty"
+        )
+      ).toThrow("column already exists");
+    });
+
+    test("dropObsColumn", async () => {
+      let xfltr;
+
+      /* check that we catch attempt to drop readonly dimension */
+      expect(() => crossfilter.dropObsColumn("louvain")).toThrow(
+        "Unknown or readonly obs column"
+      );
+      /* non-existent column */
+      expect(() => crossfilter.dropObsColumn("does-not-exist")).toThrow(
+        "Unknown or readonly obs column"
+      );
+
+      // add a column, then drop it.
+      xfltr = await helperAddTestCol(crossfilter, "foo");
+      xfltr = xfltr.dropObsColumn("foo");
+      expect(
+        xfltr.annoMatrix.schema.annotations.obs.columns.filter(
+          (v) => v.name === "foo"
+        )
+      ).toHaveLength(0);
+      expect(xfltr.annoMatrix.schema.annotations.obsByName.foo).toBeUndefined();
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(xfltr.annoMatrix.fetch("obs", "foo")).rejects.toThrow(
+        "unknown column name"
+      );
+
+      // now same, but ensure we have built an index before doing the drop
+      xfltr = await helperAddTestCol(crossfilter, "bar");
+      xfltr = await xfltr.select("obs", "bar", {
+        mode: "exact",
+        values: "whatever",
+      });
+      xfltr = xfltr.dropObsColumn("bar");
+
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(xfltr.select("obs", "bar", { mode: "all" })).rejects.toThrow(
+        "unknown column name"
+      );
+    });
+
+    test("renameObsColumn", async () => {
+      let xfltr;
+
+      /* catch attempts to rename non-existent or readonly columns */
+      expect(() =>
+        crossfilter.renameObsColumn("does-not-exist", "foo")
+      ).toThrow("Unknown or readonly obs column");
+      expect(() => crossfilter.renameObsColumn("louvain", "foo")).toThrow(
+        "Unknown or readonly obs column"
+      );
+
+      // add a column, then rename it.
+      xfltr = await helperAddTestCol(crossfilter, "foo");
+      xfltr = xfltr.renameObsColumn("foo", "bar");
+      expect(xfltr.annoMatrix.getColumnSchema("obs", "foo")).toBeUndefined();
+      expect(xfltr.annoMatrix.getColumnSchema("obs", "bar")).toMatchObject({
+        name: "bar",
+        type: "categorical",
+      });
+
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(xfltr.annoMatrix.fetch("obs", "foo")).rejects.toThrow(
+        "unknown column name"
+      );
+      const df = await xfltr.annoMatrix.fetch("obs", "bar");
+      expect(df.hasCol("bar")).toBeTruthy();
+
+      // now same, but ensure we have built an index before doing the rename
+      xfltr = await helperAddTestCol(crossfilter, "bar");
+      xfltr = await xfltr.select("obs", "bar", {
+        mode: "exact",
+        values: "whatever",
+      });
+      xfltr = xfltr.renameObsColumn("bar", "xyz");
+
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(xfltr.select("obs", "bar", { mode: "all" })).rejects.toThrow(
+        "unknown column name"
+      );
+      await expect(
+        xfltr.select("obs", "xyz", { mode: "none" })
+      ).resolves.toBeInstanceOf(AnnoMatrixObsCrossfilter);
+    });
+
+    test("addObsAnnoCategory", async () => {
+      let xfltr;
+
+      // catch unknown or readonly columns
+      expect(() => crossfilter.addObsAnnoCategory("louvain", "mumble")).toThrow(
+        "Unknown or readonly obs column"
+      );
+      expect(() =>
+        crossfilter.addObsAnnoCategory("undefined-name", "mumble")
+      ).toThrow("Unknown or readonly obs column");
+
+      // add a column and then add category to it
+      xfltr = await helperAddTestCol(crossfilter, "foo", {
+        name: "foo",
+        type: "categorical",
+        categories: ["unassigned"],
+      });
+      xfltr = xfltr.addObsAnnoCategory("foo", "a-new-label");
+      expect(xfltr.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining(["a-new-label", "unassigned"]),
+      });
+
+      // do it again, dup; should throw
+      expect(() => xfltr.addObsAnnoCategory("foo", "a-new-label")).toThrow(
+        "category already exists"
+      );
+
+      // now same, but ensure we have built an index before doing the operation
+      xfltr = await helperAddTestCol(crossfilter, "bar", {
+        name: "bar",
+        type: "categorical",
+        categories: ["unassigned"],
+      });
+      xfltr = await xfltr.select("obs", "bar", {
+        mode: "exact",
+        values: "something",
+      });
+      xfltr = xfltr.addObsAnnoCategory("bar", "a-new-label");
+      expect(xfltr.annoMatrix.getColumnSchema("obs", "bar")).toMatchObject({
+        name: "bar",
+        type: "categorical",
+        categories: expect.arrayContaining(["a-new-label", "unassigned"]),
+      });
+    });
+
+    test("removeObsAnnoCategory", async () => {
+      let xfltr;
+
+      // catch unknown or readonly categories
+      await expect(() =>
+        crossfilter.removeObsAnnoCategory("louvain", "mumble", "unassigned")
+      ).rejects.toThrow("Unknown or readonly obs column");
+      await expect(() =>
+        crossfilter.removeObsAnnoCategory("undefined-name", "mumble")
+      ).rejects.toThrow("Unknown or readonly obs column");
+
+      xfltr = await helperAddTestCol(crossfilter, "foo", {
+        name: "foo",
+        type: "categorical",
+        categories: ["unassigned", "red", "green", "blue"],
+      });
+      xfltr = await xfltr.select("obs", "foo", { mode: "all" });
+      expect(
+        (await xfltr.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .every((v) => v === "unassigned")
+      ).toBeTruthy();
+      expect(xfltr.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining([
+          "unassigned",
+          "red",
+          "green",
+          "blue",
+        ]),
+      });
+
+      // remove an unused category
+      const xfltr1 = await xfltr.removeObsAnnoCategory("foo", "red", "mumble");
+      expect(
+        (await xfltr1.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .every((v) => v === "unassigned")
+      ).toBeTruthy();
+      expect(xfltr1.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining([
+          "unassigned",
+          "green",
+          "blue",
+          "mumble",
+        ]),
+      });
+
+      // remove a used category
+      const xfltr2 = await xfltr.removeObsAnnoCategory(
+        "foo",
+        "unassigned",
+        "red"
+      );
+      expect(
+        (await xfltr2.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .every((v) => v === "red")
+      ).toBeTruthy();
+      expect(xfltr2.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining(["green", "blue", "red"]),
+      });
+    });
+
+    test("setObsColumnValues", async () => {
+      // catch unknown or readonly categories
+      await expect(() =>
+        crossfilter.setObsColumnValues("louvain", [0, 1], "unassigned")
+      ).rejects.toThrow("Unknown or readonly obs column");
+      await expect(() =>
+        crossfilter.setObsColumnValues("undefined-name", [0], "mumble")
+      ).rejects.toThrow("Unknown or readonly obs column");
+
+      let xfltr = await helperAddTestCol(crossfilter, "foo", {
+        name: "foo",
+        type: "categorical",
+        categories: ["unassigned", "red", "green", "blue"],
+      });
+      xfltr = await xfltr.select("obs", "foo", { mode: "all" });
+
+      // catch unknown row label
+      await expect(() =>
+        xfltr.setObsColumnValues("foo", [-1], "red")
+      ).rejects.toThrow("Unknown row label");
+
+      // set a few rows
+      expect(
+        (await xfltr.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .every((v) => v === "unassigned")
+      ).toBeTruthy();
+      const xfltr1 = await xfltr.setObsColumnValues("foo", [0, 10], "purple");
+      expect(
+        (await xfltr1.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .every(
+            (v, i) =>
+              v === "unassigned" || (v === "purple" && (i === 0 || i === 10))
+          )
+      ).toBeTruthy();
+      expect(xfltr1.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining([
+          "unassigned",
+          "red",
+          "green",
+          "blue",
+          "purple",
+        ]),
+      });
+
+      expect(xfltr1.countSelected()).toEqual(xfltr1.annoMatrix.nObs);
+      const xfltr2 = await xfltr1.select("obs", "foo", {
+        mode: "exact",
+        values: ["purple"],
+      });
+      expect(xfltr2.countSelected()).toEqual(2);
+      expect(xfltr2.allSelectedLabels()).toEqual(Int32Array.from([0, 10]));
+    });
+
+    test("resetObsColumnValues", async () => {
+      // catch unknown or readonly categories
+      await expect(() =>
+        crossfilter.resetObsColumnValues("louvain", "red", "blue")
+      ).rejects.toThrow("Unknown or readonly obs column");
+      await expect(() =>
+        crossfilter.resetObsColumnValues("undefined-name", "red", "blue")
+      ).rejects.toThrow("Unknown or readonly obs column");
+
+      let xfltr = await helperAddTestCol(crossfilter, "foo", {
+        name: "foo",
+        type: "categorical",
+        categories: ["unassigned", "red", "green", "blue"],
+      });
+      xfltr = await xfltr.select("obs", "foo", {
+        mode: "exact",
+        values: "red",
+      });
+
+      // catch unknown category name label
+      await expect(() =>
+        xfltr.resetObsColumnValues("foo", "unknown-label", "red")
+      ).rejects.toThrow("unknown category");
+
+      let xfltr1 = await xfltr.setObsColumnValues("foo", [0, 10], "purple");
+      xfltr1 = await xfltr1.select("obs", "foo", {
+        mode: "exact",
+        values: "purple",
+      });
+      expect(
+        (await xfltr1.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .filter((v) => v === "purple")
+      ).toHaveLength(2);
+
+      xfltr1 = await xfltr1.resetObsColumnValues("foo", "purple", "magenta");
+      expect(
+        (await xfltr1.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .filter((v) => v === "magenta")
+      ).toHaveLength(2);
+      expect(
+        (await xfltr1.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .filter((v) => v === "purple")
+      ).toHaveLength(0);
+      expect(xfltr1.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining([
+          "unassigned",
+          "red",
+          "green",
+          "blue",
+          "purple",
+          "magenta",
+        ]),
+      });
+    });
+  });
+
+  describe("edge cases", () => {
+    test("transition from empty annoMatrix", async () => {
+      // select before fetch needs to work
+      fetch.once(serverMocks.dataframeResponse(["louvain"], [obsLouvain]));
+      const xfltr = await crossfilter.select("obs", "louvain", {
+        mode: "exact",
+        values: "B cells",
+      });
+      expect(fetch.mock.calls).toHaveLength(1);
+      expect(xfltr.obsCrossfilter.hasDimension("obs/louvain")).toBeTruthy();
+      expect(xfltr.obsCrossfilter.all()).toBe(xfltr.annoMatrix._cache.obs);
+      expect(xfltr.countSelected()).toEqual(
+        obsLouvain.reduce(
+          (count, v) => (v === "B cells" ? count + 1 : count),
+          0
+        )
+      );
+    });
+  });
+});

client/__tests__/util/annoMatrix/serverMocks/index.js

@@ -0,0 +1,11 @@
+export const baseDataURL = "https://a.fake.url/api/v0.2";
+
+window.CELLXGENE = {
+  API: {
+    prefix: baseDataURL,
+    version: "v0.2/",
+  },
+};
+
+export { schema } from "./schema";
+export * from "./routes";

client/__tests__/util/annoMatrix/serverMocks/routes.js

@@ -0,0 +1,211 @@
+import { schema } from "./schema";
+import { Dataframe, KeyIndex } from "../../../../src/util/dataframe";
+import { encodeMatrixFBS } from "../../../../src/util/stateManager/matrix";
+
+const indexedSchema = {
+  obsByName: Object.fromEntries(
+    schema.schema.annotations.obs.columns.map((v) => [v.name, v]) ?? []
+  ),
+  varByName: Object.fromEntries(
+    schema.schema.annotations.var.columns.map((v) => [v.name, v]) ?? []
+  ),
+  embByName: Object.fromEntries(
+    schema.schema.layout.obs.map((v) => [v.name, v]) ?? []
+  ),
+};
+
+function makeMockColumn(s, length) {
+  const { type } = s;
+  switch (type) {
+    case "int32":
+      return new Int32Array(length).fill(Math.floor(99 * Math.random()));
+
+    case "string":
+      return new Array(length).fill("test");
+
+    case "float32":
+      return new Float32Array(length).fill(99 * Math.random());
+
+    case "boolean":
+      return new Array(length).fill(false);
+
+    case "categorical":
+      return new Array(length).fill(s.categories[0]);
+
+    default:
+      throw new Error("unkonwn type");
+  }
+}
+
+function getEncodedDataframe(colNames, length, colSchemas) {
+  const colIndex = new KeyIndex(colNames);
+  const columns = colSchemas.map((s) => makeMockColumn(s, length));
+  const df = new Dataframe([length, colNames.length], columns, null, colIndex);
+  const body = encodeMatrixFBS(df);
+  return body;
+}
+
+export function dataframeResponse(colNames, columns) {
+  const colIndex = new KeyIndex(colNames);
+  const df = new Dataframe(
+    [columns[0].length, colNames.length],
+    columns,
+    null,
+    colIndex
+  );
+  const body = encodeMatrixFBS(df);
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return () => Promise.resolve({ body, init: { status: 200, headers } });
+}
+
+function annotationObsResponse(request) {
+  const url = new URL(request.url);
+  const params = Array.from(url.searchParams.entries());
+  const names = params
+    .filter(([k]) => k === "annotation-name")
+    .map(([, v]) => v);
+  if (!names.every((n) => indexedSchema.obsByName[n])) {
+    return Promise.reject(new Error("bad obs annotation name in URL"));
+  }
+  const colSchemas = names.map((n) => indexedSchema.obsByName[n]);
+  const body = getEncodedDataframe(
+    names,
+    schema.schema.dataframe.nObs,
+    colSchemas
+  );
+
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return Promise.resolve({
+    body,
+    init: { status: 200, headers },
+  });
+}
+
+function annotationVarResponse(request) {
+  const url = new URL(request.url);
+  const params = Array.from(url.searchParams.entries());
+  const names = params
+    .filter(([k]) => k === "annotation-name")
+    .map(([, v]) => v);
+  if (!names.every((n) => indexedSchema.varByName[n])) {
+    return Promise.reject(new Error("bad var annotation name in URL"));
+  }
+  const colSchemas = names.map((n) => indexedSchema.varByName[n]);
+  const body = getEncodedDataframe(
+    names,
+    schema.schema.dataframe.nVar,
+    colSchemas
+  );
+
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return Promise.resolve({
+    body,
+    init: { status: 200, headers },
+  });
+}
+
+function layoutObsResponse(request) {
+  const url = new URL(request.url);
+  const params = Array.from(url.searchParams.entries());
+  const names = params.filter(([k]) => k === "layout-name").map(([, v]) => v);
+  if (!names.every((n) => indexedSchema.embByName[n])) {
+    return Promise.reject(new Error("bad layout name in URL"));
+  }
+  const dims = names.map((n) => indexedSchema.embByName[n].dims).flat();
+  const colSchemas = names
+    .map((n) => [indexedSchema.embByName[n], indexedSchema.embByName[n]])
+    .flat();
+  const body = getEncodedDataframe(
+    dims,
+    schema.schema.dataframe.nObs,
+    colSchemas
+  );
+
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return Promise.resolve({
+    body,
+    init: { status: 200, headers },
+  });
+}
+
+function dataVarResponse(request) {
+  const url = new URL(request.url);
+  const params = Array.from(url.searchParams.entries());
+
+  const colNames = params.map((v) => `${v[0]}/${v[1]}`);
+  const colSchemas = colNames.map(() => schema.schema.dataframe);
+  const body = getEncodedDataframe(
+    colNames,
+    schema.schema.dataframe.nObs,
+    colSchemas
+  );
+
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return Promise.resolve({
+    body,
+    init: { status: 200, headers },
+  });
+}
+
+export function responder(request) {
+  const url = new URL(request.url);
+  const { pathname } = url;
+  if (pathname.endsWith("/annotations/obs")) {
+    return annotationObsResponse(request);
+  }
+  if (pathname.endsWith("/annotations/var")) {
+    return annotationVarResponse(request);
+  }
+  if (pathname.endsWith("/layout/obs")) {
+    return layoutObsResponse(request);
+  }
+  if (pathname.endsWith("/data/var")) {
+    return dataVarResponse(request);
+  }
+  return Promise.reject(new Error("bad URL"));
+}
+
+export function withExpected(expectedURL, expectedParams) {
+  /*
+  Do some additional error checking
+  */
+  return (request) => {
+    // if URL is bogus, reject the promise
+    const url = new URL(request.url);
+    if (!url.pathname.endsWith(expectedURL)) {
+      return Promise.reject(new Error("Unexpected URL!"));
+    }
+    const params = Array.from(url.searchParams.entries()).sort(
+      (a, b) => a[0] < b[0]
+    );
+    expectedParams = expectedParams.slice().sort((a, b) => a[0] < b[0]);
+
+    if (
+      params.length !== expectedParams.length ||
+      !params.every(
+        (p, i) => p[0] === expectedParams[i][0] && p[1] === expectedParams[i][1]
+      )
+    ) {
+      return Promise.reject(new Error("unexpected name requested in URL"));
+    }
+
+    return responder(request);
+  };
+}
+
+export function annotationsObs(names) {
+  return withExpected(
+    "/annotations/obs",
+    names.map((name) => ["annotation-name", name])
+  );
+}

client/__tests__/util/annoMatrix/serverMocks/schema.js

@@ -0,0 +1,80 @@
+export const schema = {
+  schema: {
+    annotations: {
+      obs: {
+        columns: [
+          {
+            name: "name_0",
+            type: "string",
+            writable: false,
+          },
+          {
+            name: "n_genes",
+            type: "int32",
+            writable: false,
+          },
+          {
+            name: "percent_mito",
+            type: "float32",
+            writable: false,
+          },
+          {
+            name: "n_counts",
+            type: "float32",
+            writable: false,
+          },
+          {
+            name: "louvain",
+            type: "string",
+            writable: false,
+          },
+        ],
+        index: "name_0",
+      },
+      var: {
+        columns: [
+          {
+            name: "name_0",
+            type: "string",
+            writable: false,
+          },
+          {
+            name: "n_cells",
+            type: "int32",
+            writable: false,
+          },
+        ],
+        index: "name_0",
+      },
+    },
+    dataframe: {
+      nObs: 2638,
+      nVar: 1838,
+      type: "float32",
+    },
+    layout: {
+      obs: [
+        {
+          dims: ["draw_graph_fr_0", "draw_graph_fr_1"],
+          name: "draw_graph_fr",
+          type: "float32",
+        },
+        {
+          dims: ["pca_0", "pca_1"],
+          name: "pca",
+          type: "float32",
+        },
+        {
+          dims: ["tsne_0", "tsne_1"],
+          name: "tsne",
+          type: "float32",
+        },
+        {
+          dims: ["umap_0", "umap_1"],
+          name: "umap",
+          type: "float32",
+        },
+      ],
+    },
+  },
+};

client/__tests__/util/annoMatrix/whereCache.test.js

@@ -0,0 +1,184 @@
+import {
+  _whereCacheGet,
+  _whereCacheCreate,
+  _whereCacheMerge,
+} from "../../../src/annoMatrix/whereCache";
+
+const schema = {};
+
+describe("whereCache", () => {
+  test("whereCacheGet - missing cache values", () => {
+    expect(
+      _whereCacheGet({}, schema, "X", {
+        field: "var",
+        column: "foo",
+        value: "bar",
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet({ X: {} }, schema, "X", {
+        field: "var",
+        column: "foo",
+        value: "bar",
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet({ X: { var: new Map() } }, schema, "X", {
+        field: "var",
+        column: "foo",
+        value: "bar",
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet(
+        { X: { var: new Map([["foo", new Map()]]) } },
+        schema,
+        "X",
+        {
+          field: "var",
+          column: "foo",
+          value: "bar",
+        }
+      )
+    ).toEqual([undefined]);
+  });
+
+  test("whereCacheGet - varied lookups", () => {
+    const whereCache = {
+      X: {
+        var: new Map([
+          [
+            "foo",
+            new Map([
+              ["bar", [0]],
+              ["baz", [1, 2]],
+            ]),
+          ],
+        ]),
+      },
+    };
+
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        field: "var",
+        column: "foo",
+        value: "bar",
+      })
+    ).toEqual([0]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        field: "var",
+        column: "foo",
+        value: "baz",
+      })
+    ).toEqual([1, 2]);
+    expect(_whereCacheGet(whereCache, schema, "Y", {})).toEqual([undefined]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        field: "whoknows",
+        column: "whatever",
+        value: "snork",
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        field: "var",
+        column: "whatever",
+        value: "snork",
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        field: "var",
+        column: "foo",
+        value: "snork",
+      })
+    ).toEqual([undefined]);
+  });
+
+  test("whereCacheCreate", () => {
+    const query = {
+      field: "queryField",
+      column: "queryColumn",
+      value: "queryValue",
+    };
+    const wc = _whereCacheCreate(
+      "field",
+      { field: "queryField", column: "queryColumn", value: "queryValue" },
+      [0, 1, 2]
+    );
+    expect(wc).toBeDefined();
+    expect(wc).toEqual(
+      expect.objectContaining({
+        field: {
+          queryField: expect.any(Map),
+        },
+      })
+    );
+    expect(wc.field.queryField.has("queryColumn")).toEqual(true);
+    expect(wc.field.queryField.get("queryColumn")).toBeInstanceOf(Map);
+    expect(wc.field.queryField.get("queryColumn").has("queryValue")).toEqual(
+      true
+    );
+    expect(_whereCacheGet(wc, schema, "field", query)).toEqual([0, 1, 2]);
+  });
+
+  test("whereCacheMerge", () => {
+    let wc;
+
+    // remember, will mutate dst
+    const src = _whereCacheCreate(
+      "field",
+      { field: "queryField", column: "queryColumn", value: "foo" },
+      ["foo"]
+    );
+
+    const dst1 = _whereCacheCreate(
+      "field",
+      { field: "queryField", column: "queryColumn", value: "bar" },
+      ["dst1"]
+    );
+    wc = _whereCacheMerge(dst1, src);
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        field: "queryField",
+        column: "queryColumn",
+        value: "foo",
+      })
+    ).toEqual(["foo"]);
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        field: "queryField",
+        column: "queryColumn",
+        value: "bar",
+      })
+    ).toEqual(["dst1"]);
+
+    const dst2 = _whereCacheCreate(
+      "field",
+      { field: "queryField", column: "queryColumn", value: "bar" },
+      ["dst2"]
+    );
+    wc = _whereCacheMerge(dst2, dst1, src);
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        field: "queryField",
+        column: "queryColumn",
+        value: "foo",
+      })
+    ).toEqual(["foo"]);
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        field: "queryField",
+        column: "queryColumn",
+        value: "bar",
+      })
+    ).toEqual(["dst1"]);
+
+    wc = _whereCacheMerge({}, src);
+    expect(wc).toEqual(src);
+
+    wc = _whereCacheMerge({ field: { queryField: new Map() } }, src);
+    expect(wc).toEqual(src);
+  });
+});

client/__tests__/util/centroid.test.js

@@ -0,0 +1,75 @@
+import _ from "lodash";
+
+import calcCentroid from "../../src/util/centroid";
+import quantile from "../../src/util/quantile";
+import { matrixFBSToDataframe } from "../../src/util/stateManager/matrix";
+import * as REST from "./stateManager/sampleResponses";
+import { indexEntireSchema } from "../../src/util/stateManager/schemaHelpers";
+import { _normalizeCategoricalSchema } from "../../src/annoMatrix/schema";
+
+describe("centroid", () => {
+  let schema;
+  let obsAnnotations;
+  let obsLayout;
+
+  beforeAll(() => {
+    schema = indexEntireSchema(_.cloneDeep(REST.schema.schema));
+    obsAnnotations = matrixFBSToDataframe(REST.annotationsObs);
+    obsLayout = matrixFBSToDataframe(REST.layoutObs);
+
+    _normalizeCategoricalSchema(
+      schema.annotations.obsByName.field3,
+      obsAnnotations.col("field3")
+    );
+  });
+
+  test("field4 (categorical obsAnnotation)", () => {
+    const centroidResult = calcCentroid(
+      schema,
+      "field4",
+      obsAnnotations,
+      { current: "umap", currentDimNames: ["umap_0", "umap_1"] },
+      obsLayout
+    );
+
+    // Check to see that a centroid has been calculated for every categorical value
+    const keysAsArray = Array.from(centroidResult.keys());
+    expect(keysAsArray).toEqual(
+      expect.arrayContaining([83, true, "foo", 2.222222])
+    );
+
+    // This expected result assumes that all cells belong in all categorical values inside of sample response
+    const expectedResult = [
+      quantile([0.5], obsLayout.col("umap_0").asArray())[0],
+      quantile([0.5], obsLayout.col("umap_1").asArray())[0],
+    ];
+
+    centroidResult.forEach((coordinate) => {
+      expect(coordinate).toEqual(expectedResult);
+    });
+  });
+
+  test("field3 (boolean obsAnnotation)", () => {
+    const centroidResult = calcCentroid(
+      schema,
+      "field3",
+      obsAnnotations,
+      { current: "umap", currentDimNames: ["umap_0", "umap_1"] },
+      obsLayout
+    );
+
+    // Check to see that a centroid has been calculated for every categorical value
+    const keysAsArray = Array.from(centroidResult.keys());
+    expect(keysAsArray).toEqual(expect.arrayContaining([false, true]));
+
+    // This expected result assumes that all cells belong in all categorical values inside of sample response
+    const expectedResult = [
+      quantile([0.5], obsLayout.col("umap_0").asArray())[0],
+      quantile([0.5], obsLayout.col("umap_1").asArray())[0],
+    ];
+
+    centroidResult.forEach((coordinate) => {
+      expect(coordinate).toEqual(expectedResult);
+    });
+  });
+});

client/__tests__/util/dataframe/histogram.test.js

@@ -14,7 +14,7 @@ describe("Dataframe column histogram", () => {
       new Map([
         ["n1", new Map([["c1", 1]])],
         ["n2", new Map([["c2", 1]])],
-        ["n3", new Map([["c3", 1]])]
+        ["n3", new Map([["c3", 1]])],
       ])
     );
     // memoized?
@@ -31,7 +31,11 @@ describe("Dataframe column histogram", () => {
 
     const h1 = df.col("value").histogram(3, [0, 2], df.col("name"));
     expect(h1).toMatchObject(
-      new Map([["n1", [1, 0, 0]], ["n2", [0, 1, 0]], ["n3", [0, 0, 1]]])
+      new Map([
+        ["n1", [1, 0, 0]],
+        ["n2", [0, 1, 0]],
+        ["n3", [0, 0, 1]],
+      ])
     );
     // memoized?
     expect(df.col("value").histogram(3, [0, 2], df.col("name"))).toMatchObject(
@@ -48,7 +52,13 @@ describe("Dataframe column histogram", () => {
     );
 
     const h1 = df.col("cat").histogram();
-    expect(h1).toMatchObject(new Map([["c1", 1], ["c2", 1], ["c3", 1]]));
+    expect(h1).toMatchObject(
+      new Map([
+        ["c1", 1],
+        ["c2", 1],
+        ["c3", 1],
+      ])
+    );
     // memoized?
     expect(df.col("value").histogram(3, [0, 2])).toMatchObject(h1);
   });
@@ -87,7 +97,7 @@ describe("Dataframe column histogram", () => {
       0,
       0,
       0,
-      2
+      2,
     ]);
   });
 });

client/__tests__/util/dataframe/summarize.test.js

@@ -13,7 +13,7 @@ describe("Dataframe column summary", () => {
         categorical: true,
         categories: [],
         categoryCounts: new Map(),
-        numCategories: 0
+        numCategories: 0,
       })
     );
   });
@@ -27,7 +27,7 @@ describe("Dataframe column summary", () => {
         [true],
         new Float32Array([39.3]),
         new Int32Array([99]),
-        [1]
+        [1],
       ],
       null,
       new Dataframe.KeyIndex([
@@ -36,7 +36,7 @@ describe("Dataframe column summary", () => {
         "nameBoolean",
         "nameFloat32",
         "nameInt32",
-        "nameCategorical"
+        "nameCategorical",
       ])
     );
 
@@ -45,7 +45,7 @@ describe("Dataframe column summary", () => {
         categorical: true,
         categories: ["n1"],
         categoryCounts: new Map([["n1", 1]]),
-        numCategories: 1
+        numCategories: 1,
       })
     );
     expect(df.icol(1).summarize()).toEqual(
@@ -53,7 +53,7 @@ describe("Dataframe column summary", () => {
         categorical: true,
         categories: ["hi"],
         categoryCounts: new Map([["hi", 1]]),
-        numCategories: 1
+        numCategories: 1,
       })
     );
     expect(df.icol(2).summarize()).toEqual(
@@ -61,7 +61,7 @@ describe("Dataframe column summary", () => {
         categorical: true,
         categories: [true],
         categoryCounts: new Map([[true, 1]]),
-        numCategories: 1
+        numCategories: 1,
       })
     );
     expect(df.icol(3).summarize()).toEqual(
@@ -71,7 +71,7 @@ describe("Dataframe column summary", () => {
         max: float32Conversion(39.3),
         nan: 0,
         ninf: 0,
-        pinf: 0
+        pinf: 0,
       })
     );
     expect(df.icol(4).summarize()).toEqual(
@@ -81,7 +81,7 @@ describe("Dataframe column summary", () => {
         max: 99,
         nan: 0,
         ninf: 0,
-        pinf: 0
+        pinf: 0,
       })
     );
     expect(df.icol(5).summarize()).toEqual(
@@ -89,7 +89,7 @@ describe("Dataframe column summary", () => {
         categorical: true,
         categories: [1],
         categoryCounts: new Map([[1, 1]]),
-        numCategories: 1
+        numCategories: 1,
       })
     );
   });
@@ -103,7 +103,7 @@ describe("Dataframe column summary", () => {
         [false, true, true],
         new Float32Array([39.3, 39.3, 0]),
         new Int32Array([99, 99, 99]),
-        [1, false, "0"]
+        [1, false, "0"],
       ],
       null,
       new Dataframe.KeyIndex([
@@ -112,7 +112,7 @@ describe("Dataframe column summary", () => {
         "nameBoolean",
         "nameFloat32",
         "nameInt32",
-        "nameCategorical"
+        "nameCategorical",
       ])
     );
 
@@ -120,24 +120,34 @@ describe("Dataframe column summary", () => {
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining(["n0", "n1", "n2"]),
-        categoryCounts: new Map([["n0", 1], ["n1", 1], ["n2", 1]]),
-        numCategories: 3
+        categoryCounts: new Map([
+          ["n0", 1],
+          ["n1", 1],
+          ["n2", 1],
+        ]),
+        numCategories: 3,
       })
     );
     expect(df.icol(1).summarize()).toEqual(
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining(["hi", "bye"]),
-        categoryCounts: new Map([["hi", 2], ["bye", 1]]),
-        numCategories: 2
+        categoryCounts: new Map([
+          ["hi", 2],
+          ["bye", 1],
+        ]),
+        numCategories: 2,
       })
     );
     expect(df.icol(2).summarize()).toEqual(
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining([true, false]),
-        categoryCounts: new Map([[true, 2], [false, 1]]),
-        numCategories: 2
+        categoryCounts: new Map([
+          [true, 2],
+          [false, 1],
+        ]),
+        numCategories: 2,
       })
     );
     expect(df.icol(3).summarize()).toEqual(
@@ -147,7 +157,7 @@ describe("Dataframe column summary", () => {
         max: float32Conversion(39.3),
         nan: 0,
         ninf: 0,
-        pinf: 0
+        pinf: 0,
       })
     );
     expect(df.icol(4).summarize()).toEqual(
@@ -157,15 +167,19 @@ describe("Dataframe column summary", () => {
         max: 99,
         nan: 0,
         ninf: 0,
-        pinf: 0
+        pinf: 0,
       })
     );
     expect(df.icol(5).summarize()).toEqual(
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining([1, false, "0"]),
-        categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]),
-        numCategories: 3
+        categoryCounts: new Map([
+          [1, 1],
+          [false, 1],
+          ["0", 1],
+        ]),
+        numCategories: 3,
       })
     );
   });
@@ -181,10 +195,10 @@ describe("Dataframe column summary", () => {
           39.3,
           Number.NEGATIVE_INFINITY,
           Number.NaN,
-          Number.POSITIVE_INFINITY
+          Number.POSITIVE_INFINITY,
         ]),
         new Int32Array([99, 99, 99, 99]),
-        [1, false, "0", "0"]
+        [1, false, "0", "0"],
       ],
       null,
       new Dataframe.KeyIndex([
@@ -193,7 +207,7 @@ describe("Dataframe column summary", () => {
         "nameBoolean",
         "nameFloat32",
         "nameInt32",
-        "nameCategorical"
+        "nameCategorical",
       ])
     );
 
@@ -201,24 +215,34 @@ describe("Dataframe column summary", () => {
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining(["n0", "n1", "n2"]),
-        categoryCounts: new Map([["n0", 1], ["n1", 1], ["n2", 2]]),
-        numCategories: 3
+        categoryCounts: new Map([
+          ["n0", 1],
+          ["n1", 1],
+          ["n2", 2],
+        ]),
+        numCategories: 3,
       })
     );
     expect(df.icol(1).summarize()).toEqual(
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining(["hi", "bye"]),
-        categoryCounts: new Map([["hi", 2], ["bye", 1]]),
-        numCategories: 2
+        categoryCounts: new Map([
+          ["hi", 2],
+          ["bye", 1],
+        ]),
+        numCategories: 2,
       })
     );
     expect(df.icol(2).summarize()).toEqual(
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining([true, false]),
-        categoryCounts: new Map([[true, 2], [false, 1]]),
-        numCategories: 2
+        categoryCounts: new Map([
+          [true, 2],
+          [false, 1],
+        ]),
+        numCategories: 2,
       })
     );
     expect(df.icol(3).summarize()).toEqual(
@@ -228,7 +252,7 @@ describe("Dataframe column summary", () => {
         max: float32Conversion(39.3),
         nan: 1,
         ninf: 1,
-        pinf: 1
+        pinf: 1,
       })
     );
     expect(df.icol(4).summarize()).toEqual(
@@ -238,15 +262,19 @@ describe("Dataframe column summary", () => {
         max: 99,
         nan: 0,
         ninf: 0,
-        pinf: 0
+        pinf: 0,
       })
     );
     expect(df.icol(5).summarize()).toEqual(
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining([1, false, "0"]),
-        categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]),
-        numCategories: 3
+        categoryCounts: new Map([
+          [1, 1],
+          [false, 1],
+          ["0", 1],
+        ]),
+        numCategories: 3,
       })
     );
   });

client/__tests__/util/nameCreators.test.js

@@ -6,7 +6,7 @@ import {
   obsAnnoDimensionName,
   diffexpDimensionName,
   userDefinedDimensionName,
-  makeContinuousDimensionName
+  makeContinuousDimensionName,
 } from "../../src/util/nameCreators";
 
 describe("nameCreators", () => {
@@ -14,25 +14,25 @@ describe("nameCreators", () => {
     layoutDimensionName,
     obsAnnoDimensionName,
     diffexpDimensionName,
-    userDefinedDimensionName
+    userDefinedDimensionName,
   ];
 
   test("check for namespace isolation", () => {
     const foo = "foo";
-    nameCreators.forEach(fn => expect(fn(foo)).not.toBe(foo));
+    nameCreators.forEach((fn) => expect(fn(foo)).not.toBe(foo));
 
     const bar = "bar";
-    nameCreators.forEach(fn => expect(fn(foo)).not.toBe(fn(bar)));
+    nameCreators.forEach((fn) => expect(fn(foo)).not.toBe(fn(bar)));
 
-    nameCreators.forEach(fn => {
-      const allOtherFn = nameCreators.filter(elmnt => elmnt !== fn);
-      allOtherFn.forEach(otherFn => expect(fn(foo)).not.toBe(otherFn(foo)));
+    nameCreators.forEach((fn) => {
+      const allOtherFn = nameCreators.filter((elmnt) => elmnt !== fn);
+      allOtherFn.forEach((otherFn) => expect(fn(foo)).not.toBe(otherFn(foo)));
     });
   });
 
   test("check for legal keys", () => {
     /* need namespace creators to return strings only */
-    nameCreators.forEach(fn => expect(fn("X")).toMatch(/X/));
+    nameCreators.forEach((fn) => expect(fn("X")).toMatch(/X/));
   });
 });
 

client/__tests__/util/promiseLimit.test.js

@@ -0,0 +1,96 @@
+import PromiseLimit from "../../src/util/promiseLimit";
+import { range } from "../../src/util/range";
+
+const delay = (t) => new Promise((resolve) => setTimeout(resolve, t));
+
+describe("PromiseLimit", () => {
+  test("simple evaluation, concurrency 1", async () => {
+    const plimit = new PromiseLimit(1);
+    const result = await Promise.all([
+      plimit.add(() => Promise.resolve(1)),
+      plimit.add(() => Promise.resolve(2)),
+      plimit.add(() => Promise.resolve(3)),
+      plimit.add(() => Promise.resolve(4)),
+    ]);
+    expect(result).toEqual([1, 2, 3, 4]);
+  });
+
+  test("simple evaluation, concurrency > 1", async () => {
+    const plimit = new PromiseLimit(100);
+    const result = await Promise.all([
+      plimit.add(() => Promise.resolve(1)),
+      plimit.add(() => Promise.resolve(2)),
+      plimit.add(() => Promise.resolve(3)),
+      plimit.add(() => Promise.resolve(4)),
+    ]);
+    expect(result).toEqual([1, 2, 3, 4]);
+  });
+
+  test("eval in order of insertion", async () => {
+    const plimit = new PromiseLimit(100);
+
+    const result = await Promise.all([
+      plimit.add(() => Promise.resolve(1)),
+      plimit.add(() => Promise.resolve(2)),
+      plimit.add(() => Promise.resolve(3)),
+      plimit.add(() => Promise.resolve(4)),
+    ]);
+
+    expect(result).toEqual([1, 2, 3, 4]);
+  });
+
+  test("obeys concurrency limit", async () => {
+    const plimit = new PromiseLimit(2);
+    let running = 0;
+    let maxRunning = 0;
+
+    const callback = async () => {
+      running += 1;
+      maxRunning = running > maxRunning ? running : maxRunning;
+      await delay(100);
+      running -= 1;
+    };
+
+    await Promise.all(range(10).map((i) => plimit.add(() => callback(i))));
+
+    expect(maxRunning).toEqual(2);
+  });
+
+  test("rejection", async () => {
+    const plimit = new PromiseLimit(2);
+    const result = await Promise.all([
+      plimit.add(() => Promise.resolve("OK")),
+      // eslint-disable-next-line prefer-promise-reject-errors -- unit test
+      plimit.add(() => Promise.reject("not OK")).catch((e) => e),
+      plimit.add(() => Promise.resolve("OK")),
+      plimit
+        .add(() => {
+          throw new Error("not OK");
+        })
+        .catch((e) => e.message),
+    ]);
+    expect(result).toEqual(["OK", "not OK", "OK", "not OK"]);
+  });
+
+  test("priority queue", async () => {
+    const plimit = new PromiseLimit(1);
+
+    let finishOrder = 0;
+    const callback = () => async () => {
+      await delay(100);
+      const result = finishOrder;
+      finishOrder += 1;
+      return result;
+    };
+
+    const result = await Promise.all([
+      plimit.add(callback()),
+      plimit.priorityAdd(4, callback()),
+      plimit.priorityAdd(0, callback()),
+      plimit.priorityAdd(1, callback()),
+      plimit.priorityAdd(-1, callback()),
+    ]);
+
+    expect(result).toEqual([0, 4, 2, 3, 1]);
+  });
+});

client/__tests__/util/quantile.test.js

@@ -1,29 +1,29 @@
 import quantile from "../../src/util/quantile";
 
 describe("quantile", () => {
-	test("single q", () => {
-		const arr = new Float32Array([9, 3, 5, 6, 0]);
-		expect(quantile([1.0], arr)).toMatchObject([9]);
-		expect(quantile([0.9], arr)).toMatchObject([9]);
-		expect(quantile([0.8], arr)).toMatchObject([9]);
-		expect(quantile([0.7], arr)).toMatchObject([6]);
-		expect(quantile([0.6], arr)).toMatchObject([6]);
-		expect(quantile([0.5], arr)).toMatchObject([5]);
-		expect(quantile([0.4], arr)).toMatchObject([5]);
-		expect(quantile([0.3], arr)).toMatchObject([3]);
-		expect(quantile([0.2], arr)).toMatchObject([3]);
-		expect(quantile([0.1], arr)).toMatchObject([0]);
-		expect(quantile([0], arr)).toMatchObject([0]);
-	});
+  test("single q", () => {
+    const arr = new Float32Array([9, 3, 5, 6, 0]);
+    expect(quantile([1.0], arr)).toMatchObject([9]);
+    expect(quantile([0.9], arr)).toMatchObject([9]);
+    expect(quantile([0.8], arr)).toMatchObject([9]);
+    expect(quantile([0.7], arr)).toMatchObject([6]);
+    expect(quantile([0.6], arr)).toMatchObject([6]);
+    expect(quantile([0.5], arr)).toMatchObject([5]);
+    expect(quantile([0.4], arr)).toMatchObject([5]);
+    expect(quantile([0.3], arr)).toMatchObject([3]);
+    expect(quantile([0.2], arr)).toMatchObject([3]);
+    expect(quantile([0.1], arr)).toMatchObject([0]);
+    expect(quantile([0], arr)).toMatchObject([0]);
+  });
 
-	test("multi q", () => {
-		const arr = new Float32Array([9, 3, 5, 6, 0]);
-		expect(quantile([0, 0.25, 0.5, 0.75, 1.0], arr)).toMatchObject([
-			0,
-			3,
-			5,
-			6,
-			9
-		]);
-	});
+  test("multi q", () => {
+    const arr = new Float32Array([9, 3, 5, 6, 0]);
+    expect(quantile([0, 0.25, 0.5, 0.75, 1.0], arr)).toMatchObject([
+      0,
+      3,
+      5,
+      6,
+      9,
+    ]);
+  });
 });

client/__tests__/util/range.test.js

@@ -1,50 +1,48 @@
 import { range, rangeFill, linspace } from "../../src/util/range";
 
 describe("range", () => {
-	test("no defaults", () => {
-		expect(range(0, 3, 1)).toMatchObject([0, 1, 2]);
-	});
+  test("no defaults", () => {
+    expect(range(0, 3, 1)).toMatchObject([0, 1, 2]);
+  });
 
-	test("range(stop)", () => {
-		expect(range(3)).toMatchObject([0, 1, 2]);
-		expect(range(0)).toMatchObject([]);
-		expect(range(1)).toMatchObject([0]);
-	});
+  test("range(stop)", () => {
+    expect(range(3)).toMatchObject([0, 1, 2]);
+    expect(range(0)).toMatchObject([]);
+    expect(range(1)).toMatchObject([0]);
+  });
 
-	test("range(start,stop)", () => {
-		expect(range(0, 0)).toMatchObject([]);
-		expect(range(0, 2)).toMatchObject([0, 1]);
-		expect(range(4, 8)).toMatchObject([4, 5, 6, 7]);
-	});
+  test("range(start,stop)", () => {
+    expect(range(0, 0)).toMatchObject([]);
+    expect(range(0, 2)).toMatchObject([0, 1]);
+    expect(range(4, 8)).toMatchObject([4, 5, 6, 7]);
+  });
 
-	test("range(start, stop, step", () => {
-		expect(range(4, 0, -1)).toMatchObject([4, 3, 2, 1]);
-		expect(range(0, 4, 2)).toMatchObject([0, 2]);
-	});
+  test("range(start, stop, step", () => {
+    expect(range(4, 0, -1)).toMatchObject([4, 3, 2, 1]);
+    expect(range(0, 4, 2)).toMatchObject([0, 2]);
+  });
 });
 
 describe("rangefill", () => {
-	test("rangeFill(arr)", () => {
-		expect(rangeFill(new Int32Array(3))).toMatchObject(
-			new Int32Array([0, 1, 2])
-		);
-	});
-	test("rangeFill(arr, start)", () => {
-		expect(rangeFill(new Int32Array(2), 1)).toMatchObject(
-			new Int32Array([1, 2])
-		);
-	});
-	test("rangeFill(arr, start, step)", () => {
-		expect(rangeFill(new Int32Array(3), 2, -1)).toMatchObject(
-			new Int32Array([2, 1, 0])
-		);
-	});
+  test("rangeFill(arr)", () => {
+    expect(rangeFill(new Int32Array(3))).toMatchObject(
+      new Int32Array([0, 1, 2])
+    );
+  });
+  test("rangeFill(arr, start)", () => {
+    expect(rangeFill(new Int32Array(2), 1)).toMatchObject(
+      new Int32Array([1, 2])
+    );
+  });
+  test("rangeFill(arr, start, step)", () => {
+    expect(rangeFill(new Int32Array(3), 2, -1)).toMatchObject(
+      new Int32Array([2, 1, 0])
+    );
+  });
 });
 
 describe("linspace", () => {
-	test("linspace(arr, start, step)", () => {
-		expect(linspace(0.0, 2.0, 5)).toMatchObject(
-		    [0.0, 0.5, 1.0, 1.5, 2.0]
-		);
-	});
+  test("linspace(arr, start, step)", () => {
+    expect(linspace(0.0, 2.0, 5)).toMatchObject([0.0, 0.5, 1.0, 1.5, 2.0]);
+  });
 });

client/__tests__/util/stateManager/controlsHelpers.test.js

@@ -0,0 +1,38 @@
+/*
+test controls helpers
+*/
+import { subsetAndResetGeneLists } from "../../../src/util/stateManager/controlsHelpers";
+import * as globals from "../../../src/globals";
+
+describe("controls helpers", () => {
+  test("subsetAndResetGeneLists", () => {
+    const geneList = [];
+    const genRandGene = () => Math.random().toString(36).substring(2, 6);
+
+    // build a unique set of genes
+    for (let i = 0; i < 150; i += 1) {
+      let randGene = genRandGene();
+      while (geneList.includes(randGene)) randGene = genRandGene();
+      geneList.push(randGene);
+    }
+    // insert duplicates
+    geneList[0] = "dupl";
+    geneList[20] = "dupl";
+    const state = {
+      userDefinedGenes: geneList.slice(0, 20),
+      diffexpGenes: geneList.slice(20),
+    };
+    const [newUserDefinedGenes, newDiffExpGenes] = subsetAndResetGeneLists(
+      state
+    );
+    const expectedNewUserDefinedGenes = [
+      ...geneList.slice(0, 20),
+      ...geneList.slice(21),
+    ].slice(0, globals.maxGenes);
+    expect(globals.maxUserDefinedGenes).toBeLessThan(globals.maxGenes);
+    expect(geneList.length).toBeGreaterThan(globals.maxGenes);
+    expect(newUserDefinedGenes).toHaveLength(globals.maxGenes);
+    expect(newUserDefinedGenes).toStrictEqual(expectedNewUserDefinedGenes);
+    expect(newDiffExpGenes).toStrictEqual([]);
+  });
+});

client/__tests__/util/stateManager/fbs.test.js

@@ -3,32 +3,32 @@ test FBS encode/decode API
 */
 import { Dataframe, KeyIndex } from "../../../src/util/dataframe";
 import {
-	decodeMatrixFBS,
-	encodeMatrixFBS
+  decodeMatrixFBS,
+  encodeMatrixFBS,
 } from "../../../src/util/stateManager/matrix";
 
 describe("encode/decode", () => {
-	test("round trip", () => {
-		const columns = [
-			["red", "green", "blue"],
-			new Int32Array(3).fill(0),
-			new Uint32Array(3).fill(1),
-			new Float32Array(3).fill(2)
-		];
+  test("round trip", () => {
+    const columns = [
+      ["red", "green", "blue"],
+      new Int32Array(3).fill(0),
+      new Uint32Array(3).fill(1),
+      new Float32Array(3).fill(2),
+    ];
 
-		const dfNoColIdx = new Dataframe([3, 4], columns);
-		const dfA = decodeMatrixFBS(encodeMatrixFBS(dfNoColIdx));
-		expect([dfA.nRows, dfA.nCols]).toEqual(dfNoColIdx.dims);
-		expect(dfA.colIdx).toBeNull();
-		expect(dfA.rowIdx).toBeNull();
-		expect(dfA.columns).toEqual(columns);
+    const dfNoColIdx = new Dataframe([3, 4], columns);
+    const dfA = decodeMatrixFBS(encodeMatrixFBS(dfNoColIdx));
+    expect([dfA.nRows, dfA.nCols]).toEqual(dfNoColIdx.dims);
+    expect(dfA.colIdx).toBeNull();
+    expect(dfA.rowIdx).toBeNull();
+    expect(dfA.columns).toEqual(columns);
 
-		const colIndex = new KeyIndex(["a", "b", "c", "d"]);
-		const dfWithColIdx = new Dataframe([3, 4], columns, null, colIndex);
-		const dfB = decodeMatrixFBS(encodeMatrixFBS(dfWithColIdx));
-		expect([dfB.nRows, dfB.nCols]).toEqual(dfWithColIdx.dims);
-		expect(dfB.colIdx).toEqual(colIndex.keys());
-		expect(dfB.rowIdx).toBeNull();
-		expect(dfB.columns).toEqual(columns);
-	});
+    const colIndex = new KeyIndex(["a", "b", "c", "d"]);
+    const dfWithColIdx = new Dataframe([3, 4], columns, null, colIndex);
+    const dfB = decodeMatrixFBS(encodeMatrixFBS(dfWithColIdx));
+    expect([dfB.nRows, dfB.nCols]).toEqual(dfWithColIdx.dims);
+    expect(dfB.colIdx).toEqual(colIndex.labels());
+    expect(dfB.rowIdx).toBeNull();
+    expect(dfB.columns).toEqual(columns);
+  });
 });

client/__tests__/util/stateManager/sampleResponses.js

@@ -1,4 +1,3 @@
-/* eslint no-bitwise: "off" */
 import _ from "lodash";
 import { flatbuffers } from "flatbuffers";
 import { NetEncoding } from "../../../src/util/stateManager/matrix_generated";
@@ -17,13 +16,13 @@ const aConfigResponse = {
       { method: "POST", path: "/cluster/", available: false },
       { method: "POST", path: "/layout/", available: false },
       { method: "POST", path: "/diffexp/", available: false },
-      { method: "POST", path: "/saveLocal/", available: false }
+      { method: "POST", path: "/saveLocal/", available: false },
     ],
     displayNames: {
       engine: "the little engine that could",
-      dataset: "all your zeros are mine"
-    }
-  }
+      dataset: "all your zeros are mine",
+    },
+  },
 };
 
 const aSchemaResponse = {
@@ -31,7 +30,7 @@ const aSchemaResponse = {
     dataframe: {
       nObs,
       nVar,
-      type: "float32"
+      type: "float32",
     },
     annotations: {
       obs: {
@@ -44,9 +43,9 @@ const aSchemaResponse = {
           {
             name: "field4",
             type: "categorical",
-            categories: field4Categories
-          }
-        ]
+            categories: field4Categories,
+          },
+        ],
       },
       var: {
         index: "name",
@@ -58,46 +57,48 @@ const aSchemaResponse = {
           {
             name: "fieldD",
             type: "categorical",
-            categories: fieldDCategories
-          }
-        ]
-      }
+            categories: fieldDCategories,
+          },
+        ],
+      },
     },
     layout: {
       obs: [{ name: "umap", type: "float32", dims: ["umap_0", "umap_1"] }],
-      var: []
-    }
-  }
+      var: [],
+    },
+  },
 };
 
 const anAnnotationsObsJSONResponse = {
   names: ["name", "field1", "field2", "field3", "field4"],
   data: _()
     .range(nObs)
-    .map(idx => [
+    .map((idx) => [
       idx,
       `obs${idx}`,
       2 * idx,
       idx + 0.0133,
+      // eslint-disable-next-line no-bitwise -- idx & 1 to check for odd numbers
       !!(idx & 1),
-      field4Categories[idx % field4Categories.length]
+      field4Categories[idx % field4Categories.length],
     ])
-    .value()
+    .value(),
 };
 
 const anAnnotationsVarJSONResponse = {
   names: ["fieldA", "fieldB", "fieldC", "fieldD", "name"],
   data: _()
     .range(nVar)
-    .map(idx => [
+    .map((idx) => [
       idx,
       10 * idx,
       idx + 2.90143,
+      // eslint-disable-next-line no-bitwise -- idx & 1 to check for odd numbers
       !!(idx & 1),
       fieldDCategories[idx % fieldDCategories.length],
-      `var${idx}`
+      `var${idx}`,
     ])
-    .value()
+    .value(),
 };
 
 function encodeTypedArray(builder, uType, uData) {
@@ -112,7 +113,7 @@ function encodeTypedArray(builder, uType, uData) {
 function encodeMatrix(columns, colIndex = undefined) {
   /*
   IMPORTANT: this is not a general purpose encoder.  in particular,
-  it doesn't correctly handle all column index types, nor does it 
+  it doesn't correctly handle all column index types, nor does it
   handle all column typedarray types.
 
   encodeMatrixFBS in matrix.py is more general.  This is used only
@@ -120,7 +121,7 @@ function encodeMatrix(columns, colIndex = undefined) {
   */
   const utf8Encoder = new TextEncoder("utf-8");
   const builder = new flatbuffers.Builder(1024);
-  const cols = _.map(columns, carr => {
+  const cols = _.map(columns, (carr) => {
     let uType;
     let tarr;
     if (_.every(carr, _.isNumber)) {
@@ -178,7 +179,7 @@ const anAnnotationsVarFBSResponse = (() => {
 const aLayoutFBSResponse = (() => {
   const coords = [
     new Float32Array(nObs).fill(Math.random()),
-    new Float32Array(nObs).fill(Math.random())
+    new Float32Array(nObs).fill(Math.random()),
   ];
   return encodeMatrix(coords, ["umap_0", "umap_1"]);
 })();
@@ -187,8 +188,8 @@ const aDataObsResponse = {
   var: [2, 4, 29],
   obs: _()
     .range(nObs)
-    .map(idx => [idx, Math.random(), Math.random(), Math.random()])
-    .value()
+    .map((idx) => [idx, Math.random(), Math.random(), Math.random()])
+    .value(),
 };
 
 export {
@@ -197,5 +198,5 @@ export {
   anAnnotationsVarFBSResponse as annotationsVar,
   anAnnotationsObsFBSResponse as annotationsObs,
   aSchemaResponse as schema,
-  aConfigResponse as config
+  aConfigResponse as config,
 };

client/__tests__/util/stateManager/universe.test.js

@@ -1,68 +0,0 @@
-import * as Universe from "../../../src/util/stateManager/universe";
-import * as Dataframe from "../../../src/util/dataframe";
-import * as REST from "./sampleResponses";
-
-describe("createUniverseFromResponse", () => {
-  /*
-  test createUniverseFromResponse - this function converts
-  a set of REST 0.2 responses into a "new" Universe.
-
-  createUniverseFromResponse(
-    configResponse,
-    schemaResponse,
-    annotationsObsResponse,
-    annotationsVarResponse,
-    layoutObsResponse
-  ) --> Universe
-
-  where:
-  configResponse: GET /.../config
-  schemaResponse: GET /.../schema
-  annotationsObsResponse: GET /.../annotations/obs
-  annotationsVarResponse: GET /.../annotations/var
-  layoutObsResponse: GET /.../layout/obs
-
-  See spec in docs/REST_API.md.
-  */
-
-  test("create from test data", () => {
-    /*
-    create a universe from sample data nad validate its shape & contents
-    */
-    const { nObs, nVar } = REST.schema.schema.dataframe;
-    const universe = Universe.createUniverseFromResponse(
-      REST.config,
-      REST.schema,
-      REST.annotationsObs,
-      REST.annotationsVar,
-      REST.layoutObs
-    );
-
-    expect(universe).toBeDefined();
-    expect(universe).toMatchObject(
-      expect.objectContaining({
-        nObs,
-        nVar,
-        schema: REST.schema.schema,
-        obsAnnotations: expect.any(Dataframe.Dataframe),
-        varAnnotations: expect.any(Dataframe.Dataframe),
-        obsLayout: expect.any(Dataframe.Dataframe),
-        varData: expect.any(Dataframe.Dataframe)
-      })
-    );
-
-    expect(universe.obsAnnotations.dims).toEqual([
-      nObs,
-      REST.schema.schema.annotations.obs.columns.length
-    ]);
-    expect(universe.obsLayout.dims).toEqual([nObs, 2]);
-    expect(universe.obsLayout.colIndex.keys()).toEqual(
-      universe.schema.layout.obs[0].dims
-    );
-    expect(universe.varAnnotations.dims).toEqual([
-      nVar,
-      REST.schema.schema.annotations.var.columns.length
-    ]);
-    expect(universe.varData.isEmpty()).toBeTruthy();
-  });
-});

client/__tests__/util/stateManager/world.test.js

@@ -1,190 +0,0 @@
-import _ from "lodash";
-import * as Universe from "../../../src/util/stateManager/universe";
-import * as World from "../../../src/util/stateManager/world";
-import * as Dataframe from "../../../src/util/dataframe";
-import Crossfilter from "../../../src/util/typedCrossfilter";
-import { DimTypes } from "../../../src/util/typedCrossfilter/crossfilter";
-import * as REST from "./sampleResponses";
-import {
-  obsAnnoDimensionName,
-  layoutDimensionName
-} from "../../../src/util/nameCreators";
-
-/*
-Helper - creates universe, world, corssfilter and dimensionMap from
-the default REST test response.
-*/
-const defaultBigBang = () => {
-  /* create unverse, world, crossfilter and dimensionMap */
-  /* create universe */
-  const universe = Universe.createUniverseFromResponse(
-    _.cloneDeep(REST.config),
-    _.cloneDeep(REST.schema),
-    _.cloneDeep(REST.annotationsObs),
-    _.cloneDeep(REST.annotationsVar),
-    _.cloneDeep(REST.layoutObs)
-  );
-  /* create world */
-  const world = World.createWorldFromEntireUniverse(universe);
-  /* create crossfilter */
-  const crossfilter = World.createObsDimensions(
-    new Crossfilter(world.obsAnnotations),
-    world,
-    REST.schema.schema.layout.obs[0].dims
-  );
-
-  return {
-    universe,
-    world,
-    crossfilter
-  };
-};
-
-describe("createWorldFromEntireUniverse", () => {
-  test("create from REST sample", () => {
-    const universe = Universe.createUniverseFromResponse(
-      _.cloneDeep(REST.config),
-      _.cloneDeep(REST.schema),
-      _.cloneDeep(REST.annotationsObs),
-      _.cloneDeep(REST.annotationsVar),
-      _.cloneDeep(REST.layoutObs)
-    );
-    expect(universe).toBeDefined();
-
-    const world = World.createWorldFromEntireUniverse(universe);
-    expect(world).toBeDefined();
-
-    expect(world).toMatchObject(
-      expect.objectContaining({
-        nObs: universe.nObs,
-        nVar: universe.nVar,
-        schema: universe.schema,
-        obsAnnotations: expect.any(Dataframe.Dataframe),
-        varAnnotations: expect.any(Dataframe.Dataframe),
-        obsLayout: expect.any(Dataframe.Dataframe),
-        varData: expect.any(Dataframe.Dataframe),
-        clipQuantiles: { min: 0, max: 1 },
-        unclipped: {
-          obsAnnotations: expect.any(Dataframe.Dataframe),
-          varData: expect.any(Dataframe.Dataframe)
-        }
-      })
-    );
-  });
-});
-
-describe("createWorldFromCurrentSelection", () => {
-  test("create from REST sample", () => {
-    const {
-      universe,
-      world: originalWorld,
-      crossfilter: originalCrossfilter
-    } = defaultBigBang();
-
-    /* mock a selection */
-    const crossfilter = originalCrossfilter
-      .select(obsAnnoDimensionName("field1"), { mode: "range", lo: 0, hi: 5 })
-      .select(obsAnnoDimensionName("field3"), {
-        mode: "exact",
-        values: [false]
-      });
-
-    /* create the world from the selection */
-    const world = World.createWorldBySelection(
-      universe,
-      originalWorld,
-      crossfilter
-    );
-    expect(world).toBeDefined();
-    expect(world.nObs).toEqual(crossfilter.countSelected());
-
-    /*
-    calculate expected values and match against result
-    */
-
-    /* matchFilter must match the dimension filters above */
-    const matchFilter = (df, row) => {
-      const field1 = df.at(row, "field1");
-      const field3 = df.at(row, "field3");
-      return field1 >= 0 && field1 < 5 && !field3;
-    };
-    const matchingIndices = _()
-      .range(universe.nObs)
-      .filter(idx => matchFilter(universe.obsAnnotations, idx))
-      .value();
-
-    expect(world).toMatchObject(
-      expect.objectContaining({
-        nObs: matchingIndices.length,
-        nVar: universe.nVar,
-        schema: universe.schema,
-        clipQuantiles: { min: 0, max: 1 },
-        obsAnnotations: expect.any(Dataframe.Dataframe),
-        varAnnotations: expect.any(Dataframe.Dataframe),
-        obsLayout: expect.any(Dataframe.Dataframe),
-        varData: expect.any(Dataframe.Dataframe),
-        unclipped: {
-          obsAnnotations: expect.any(Dataframe.Dataframe),
-          varData: expect.any(Dataframe.Dataframe)
-        }
-      })
-    );
-
-    expect(world.obsAnnotations.rowIndex.keys()).toEqual(
-      new Int32Array(matchingIndices)
-    );
-    expect(world.obsAnnotations.colIndex.keys()).toEqual(
-      universe.obsAnnotations.colIndex.keys()
-    );
-    expect(world.obsLayout.rowIndex.keys()).toEqual(
-      new Int32Array(matchingIndices)
-    );
-    expect(world.obsLayout.colIndex.keys()).toEqual(
-      world.schema.layout.obs[0].dims
-    );
-  });
-});
-
-describe("createObsDimensionMap", () => {
-  test("when universe eq world", () => {
-    /*
-    check for:
-    - creates a dimension for all obsAnnotations, PLUS X/Y layout
-    - check that dimension typing is sane
-    */
-
-    const { crossfilter } = defaultBigBang();
-    const annotationNames = _.map(
-      REST.schema.schema.annotations.obs.columns,
-      c => c.name
-    );
-    const obsIndexColName = REST.schema.schema.annotations.obs.index;
-    const schemaByObsName = _.keyBy(
-      REST.schema.schema.annotations.obs.columns,
-      "name"
-    );
-    expect(crossfilter).toBeDefined();
-    annotationNames.forEach(name => {
-      const dim = crossfilter.dimensions[obsAnnoDimensionName(name)];
-      if (name === obsIndexColName) {
-        expect(dim).toBeUndefined();
-      } else {
-        const { type } = schemaByObsName[name];
-        if (type === "string" || type === "boolean" || type === "categorical") {
-          expect(dim.dim).toBeInstanceOf(DimTypes.enum);
-        } else {
-          expect(dim.dim).toBeInstanceOf(DimTypes.scalar);
-        }
-      }
-    });
-    expect(
-      crossfilter.dimensions[layoutDimensionName("XY")].dim
-    ).toBeInstanceOf(DimTypes.spatial);
-  });
-});
-
-describe("worldEqUniverse", () => {
-  const { universe, world } = defaultBigBang();
-  const result = World.worldEqUniverse(world, universe);
-  expect(result).toBe(true);
-});

client/__tests__/util/typedCrossfilter/bitArray.test.js

@@ -183,3 +183,24 @@ describe("fillBySelection", () => {
     expect(arr).toEqual(truth);
   });
 });
+
+describe("wide bitarray", () => {
+  test.each([9, 30, 31, 32, 33, 54, 63, 64, 65, 127, 128, 129])(
+    "more than %d dimensions",
+    (ndim) => {
+      /* ensure we move across the uint boundary correctly */
+
+      const ba = new BitArray(defaultTestLength);
+      expect(ba).toBeDefined();
+
+      for (let i = 0; i < ndim; i += 1) {
+        expect(ba.allocDimension()).toEqual(i);
+      }
+      ba.freeDimension(0);
+      expect(ba.allocDimension()).toEqual(0);
+
+      ba.freeDimension(ndim - 1);
+      expect(ba.allocDimension()).toEqual(ndim - 1);
+    }
+  );
+});

client/__tests__/util/typedCrossfilter/crossfilter.test.js

@@ -11,7 +11,7 @@ const someData = [
     type: "tab",
     productIDs: ["001"],
     coords: [0, 0],
-    nonFinite: 0.0
+    nonFinite: 0.0,
   },
   {
     date: "2011-11-14T16:20:19Z",
@@ -21,7 +21,7 @@ const someData = [
     type: "tab",
     productIDs: ["001", "005"],
     coords: [0.4, 0.4],
-    nonFinite: Number.NaN
+    nonFinite: Number.NaN,
   },
   {
     date: "2011-11-14T16:28:54Z",
@@ -31,7 +31,7 @@ const someData = [
     type: "visa",
     productIDs: ["004", "005"],
     coords: [0.3, 0.1],
-    nonFinite: Number.POSITIVE_INFINITY
+    nonFinite: Number.POSITIVE_INFINITY,
   },
   {
     date: "2011-11-14T16:30:43Z",
@@ -41,7 +41,7 @@ const someData = [
     type: "tab",
     productIDs: ["001", "002"],
     coords: [0.392, 0.1],
-    nonFinite: Number.NEGATIVE_INFINITY
+    nonFinite: Number.NEGATIVE_INFINITY,
   },
   {
     date: "2011-11-14T16:48:46Z",
@@ -51,7 +51,7 @@ const someData = [
     type: "tab",
     productIDs: ["005"],
     coords: [0.7, 0.0482],
-    nonFinite: 1.0
+    nonFinite: 1.0,
   },
   {
     date: "2011-11-14T16:53:41Z",
@@ -61,7 +61,7 @@ const someData = [
     type: "tab",
     productIDs: ["001", "004", "005"],
     coords: [0.9999, 1.0],
-    nonFinite: Number.NaN
+    nonFinite: Number.NaN,
   },
   {
     date: "2011-11-14T16:54:06Z",
@@ -71,7 +71,7 @@ const someData = [
     type: "cash",
     productIDs: ["001", "002", "003", "004", "005"],
     coords: [0.384, 0.6938],
-    nonFinite: 99.0
+    nonFinite: 99.0,
   },
   {
     date: "2011-11-14T16:58:03Z",
@@ -81,7 +81,7 @@ const someData = [
     type: "tab",
     productIDs: ["001"],
     coords: [0.4822, 0.482],
-    nonFinite: Number.NaN
+    nonFinite: Number.NaN,
   },
   {
     date: "2011-11-14T17:07:21Z",
@@ -91,7 +91,7 @@ const someData = [
     type: "tab",
     productIDs: ["004", "005"],
     coords: [0.2234, 0],
-    nonFinite: Number.NaN
+    nonFinite: Number.NaN,
   },
   {
     date: "2011-11-14T17:22:59Z",
@@ -101,7 +101,7 @@ const someData = [
     type: "tab",
     productIDs: ["001", "002", "004", "005"],
     coords: [0.382, 0.38485],
-    nonFinite: -1
+    nonFinite: -1,
   },
   {
     date: "2011-11-14T17:25:45Z",
@@ -111,7 +111,7 @@ const someData = [
     type: "cash",
     productIDs: ["002"],
     coords: [0.998, 0.8472],
-    nonFinite: 0.0
+    nonFinite: 0.0,
   },
   {
     date: "2011-11-14T17:29:52Z",
@@ -121,8 +121,8 @@ const someData = [
     type: "visa",
     productIDs: ["004"],
     coords: [0.8273, 0.3384],
-    nonFinite: 0.0
-  }
+    nonFinite: 0.0,
+  },
 ];
 
 let payments = null;
@@ -248,16 +248,26 @@ describe("ImmutableTypedCrossfilter", () => {
     test("none", () => {
       expect(p.select("quantity", { mode: "none" }).countSelected()).toEqual(0);
     });
-    test.each([[[]], [[2]], [[2, 1]], [[9, 82]], [[0, 1]]])("exact: %p", v =>
+    test.each([[[]], [[2]], [[2, 1]], [[9, 82]], [[0, 1]]])("exact: %p", (v) =>
       expect(
         p.select("quantity", { mode: "exact", values: v }).countSelected()
-      ).toEqual(_.filter(someData, d => v.includes(d.quantity)).length)
+      ).toEqual(_.filter(someData, (d) => v.includes(d.quantity)).length)
     );
-    test.each([[0, 1], [1, 2], [0, 99], [99, 100000]])("range %p", (lo, hi) =>
+    test("single value exact", () => {
+      expect(
+        p.select("quantity", { mode: "exact", values: 2 }).countSelected()
+      ).toEqual(_.filter(someData, (d) => d.quantity === 2).length);
+    });
+    test.each([
+      [0, 1],
+      [1, 2],
+      [0, 99],
+      [99, 100000],
+    ])("range %p", (lo, hi) =>
       expect(
         p.select("quantity", { mode: "range", lo, hi }).countSelected()
       ).toEqual(
-        _.filter(someData, d => d.quantity >= lo && d.quantity < hi).length
+        _.filter(someData, (d) => d.quantity >= lo && d.quantity < hi).length
       )
     );
     test("bad mode", () => {
@@ -284,12 +294,17 @@ describe("ImmutableTypedCrossfilter", () => {
       [["tab"]],
       [["visa"]],
       [["visa", "tab"]],
-      [["cash", "tab", "visa"]]
-    ])("exact: %p", v =>
+      [["cash", "tab", "visa"]],
+    ])("exact: %p", (v) =>
       expect(
         p.select("type", { mode: "exact", values: v }).countSelected()
-      ).toEqual(_.filter(someData, d => v.includes(d.type)).length)
+      ).toEqual(_.filter(someData, (d) => v.includes(d.type)).length)
     );
+    test("single value exact", () => {
+      expect(
+        p.select("type", { mode: "exact", values: "tab" }).countSelected()
+      ).toEqual(_.filter(someData, (d) => d.type === "tab").length);
+    });
     test("range", () => {
       expect(() => p.select("type", { mode: "range", lo: 0, hi: 9 })).toThrow(
         Error
@@ -303,8 +318,8 @@ describe("ImmutableTypedCrossfilter", () => {
   describe("spatial dimension", () => {
     let p;
     beforeEach(() => {
-      const X = someData.map(r => r.coords[0]);
-      const Y = someData.map(r => r.coords[1]);
+      const X = someData.map((r) => r.coords[0]);
+      const Y = someData.map((r) => r.coords[1]);
       p = payments.addDimension("coords", "spatial", X, Y);
     });
 
@@ -316,34 +331,76 @@ describe("ImmutableTypedCrossfilter", () => {
     test("none", () => {
       expect(p.select("coords", { mode: "none" }).countSelected()).toEqual(0);
     });
-    test.each([[0, 0, 1, 1], [0, 0, 0.5, 0.5], [0.5, 0.5, 1, 1]])(
-      "within-rect %d %d %d %d",
-      (minX, minY, maxX, maxY) => {
-        expect(
-          p
-            .select("coords", { mode: "within-rect", minX, minY, maxX, maxY })
-            .allSelected()
-        ).toEqual(
-          _.filter(someData, d => {
-            const [x, y] = d.coords;
-            return minX <= x && x < maxX && minY <= y && y < maxY;
-          })
-        );
-      }
-    );
+    test.each([
+      [0, 0, 1, 1],
+      [0, 0, 0.5, 0.5],
+      [0.5, 0.5, 1, 1],
+    ])("within-rect %d %d %d %d", (minX, minY, maxX, maxY) => {
+      expect(
+        p
+          .select("coords", { mode: "within-rect", minX, minY, maxX, maxY })
+          .allSelected()
+      ).toEqual(
+        _.filter(someData, (d) => {
+          const [x, y] = d.coords;
+          return minX <= x && x < maxX && minY <= y && y < maxY;
+        })
+      );
+    });
 
     test.each([
       [
-          [[0, 0], [0, 1], [1, 1], [1, 0]],
-          [true, true, true, true, true, false, true, true, true, true, true, true]
+        [
+          [0, 0],
+          [0, 1],
+          [1, 1],
+          [1, 0],
+        ],
+        [
+          true,
+          true,
+          true,
+          true,
+          true,
+          false,
+          true,
+          true,
+          true,
+          true,
+          true,
+          true,
+        ],
       ],
       [
-          [[0, 0], [0, 0.5], [0.5, 0.5], [0.5, 0]],
-          [true, true, true, true, false, false, false, true, true, true, false, false]
-      ]
+        [
+          [0, 0],
+          [0, 0.5],
+          [0.5, 0.5],
+          [0.5, 0],
+        ],
+        [
+          true,
+          true,
+          true,
+          true,
+          false,
+          false,
+          false,
+          true,
+          true,
+          true,
+          false,
+          false,
+        ],
+      ],
     ])("within-polygon %p", (polygon, expected) => {
-      expect(p.select("coords", { mode: "within-polygon", polygon }).allSelected())
-          .toEqual(_.zip(someData, expected).filter(x => x[1]).map(x => x[0]));
+      expect(
+        p.select("coords", { mode: "within-polygon", polygon }).allSelected()
+      ).toEqual(
+        _.zip(someData, expected)
+          .filter((x) => x[1])
+          .map((x) => x[0])
+      );
     });
   });
 
@@ -381,7 +438,7 @@ describe("ImmutableTypedCrossfilter", () => {
         p
           .select("nonFinite", {
             mode: "exact",
-            values: [Number.POSITIVE_INFINITY]
+            values: [Number.POSITIVE_INFINITY],
           })
           .countSelected()
       ).toEqual(1);
@@ -389,7 +446,7 @@ describe("ImmutableTypedCrossfilter", () => {
         p
           .select("nonFinite", {
             mode: "exact",
-            values: [Number.NEGATIVE_INFINITY]
+            values: [Number.NEGATIVE_INFINITY],
           })
           .countSelected()
       ).toEqual(1);
@@ -402,7 +459,7 @@ describe("ImmutableTypedCrossfilter", () => {
         p
           .select("nonFinite", {
             mode: "exact",
-            values: [Number.POSITIVE_INFINITY, 0, 1, 99]
+            values: [Number.POSITIVE_INFINITY, 0, 1, 99],
           })
           .countSelected()
       ).toEqual(6);
@@ -414,7 +471,7 @@ describe("ImmutableTypedCrossfilter", () => {
           .select("nonFinite", {
             mode: "range",
             lo: 0,
-            hi: Number.POSITIVE_INFINITY
+            hi: Number.POSITIVE_INFINITY,
           })
           .countSelected()
       ).toEqual(5);
@@ -423,7 +480,7 @@ describe("ImmutableTypedCrossfilter", () => {
           .select("nonFinite", {
             mode: "range",
             lo: 0,
-            hi: Number.NaN
+            hi: Number.NaN,
           })
           .countSelected()
       ).toEqual(6);
@@ -432,7 +489,7 @@ describe("ImmutableTypedCrossfilter", () => {
           .select("nonFinite", {
             mode: "range",
             lo: Number.NEGATIVE_INFINITY,
-            hi: Number.POSITIVE_INFINITY
+            hi: Number.POSITIVE_INFINITY,
           })
           .countSelected()
       ).toEqual(7);

client/__tests__/util/typedCrossfilter/positiveInterval.test.js

@@ -10,15 +10,30 @@ describe("canonicalize", () => {
 
   test("simple, already correct", () => {
     expect(PositiveIntervals.canonicalize([[0, 1]])).toEqual([[0, 1]]);
-    expect(PositiveIntervals.canonicalize([[0, 1], [2, 3]])).toEqual([
+    expect(
+      PositiveIntervals.canonicalize([
+        [0, 1],
+        [2, 3],
+      ])
+    ).toEqual([
       [0, 1],
-      [2, 3]
+      [2, 3],
     ]);
   });
 
   test("non-canonical, need to be canonicalized", () => {
-    expect(PositiveIntervals.canonicalize([[0, 1], [1, 2]])).toEqual([[0, 2]]);
-    expect(PositiveIntervals.canonicalize([[1, 2], [2, 3]])).toEqual([[1, 3]]);
+    expect(
+      PositiveIntervals.canonicalize([
+        [0, 1],
+        [1, 2],
+      ])
+    ).toEqual([[0, 2]]);
+    expect(
+      PositiveIntervals.canonicalize([
+        [1, 2],
+        [2, 3],
+      ])
+    ).toEqual([[1, 3]]);
   });
 });
 
@@ -26,14 +41,30 @@ describe("union", () => {
   test("empty range", () => {
     expect(PositiveIntervals.union([], [])).toEqual([]);
     expect(PositiveIntervals.union([], [[1, 2]])).toEqual([[1, 2]]);
-    expect(PositiveIntervals.union([], [[1, 2], [3, 4]])).toEqual([
+    expect(
+      PositiveIntervals.union(
+        [],
+        [
+          [1, 2],
+          [3, 4],
+        ]
+      )
+    ).toEqual([
       [1, 2],
-      [3, 4]
+      [3, 4],
     ]);
     expect(PositiveIntervals.union([[3, 4]], [])).toEqual([[3, 4]]);
-    expect(PositiveIntervals.union([[1, 2], [3, 4]], [])).toEqual([
+    expect(
+      PositiveIntervals.union(
+        [
+          [1, 2],
+          [3, 4],
+        ],
+        []
+      )
+    ).toEqual([
       [1, 2],
-      [3, 4]
+      [3, 4],
     ]);
     expect(PositiveIntervals.union([[3, 3]], [])).toEqual([[3, 3]]);
     expect(PositiveIntervals.union([], [[3, 3]])).toEqual([[3, 3]]);
@@ -44,17 +75,37 @@ describe("union", () => {
     expect(PositiveIntervals.union([[2, 3]], [[1, 2]])).toEqual([[1, 3]]);
     expect(PositiveIntervals.union([[1, 2]], [[3, 4]])).toEqual([
       [1, 2],
-      [3, 4]
+      [3, 4],
     ]);
     expect(
-      PositiveIntervals.union([[1, 2], [3, 4]], [[6, 7], [19, 40]])
-    ).toEqual([[1, 2], [3, 4], [6, 7], [19, 40]]);
-    expect(PositiveIntervals.union([[1, 4]], [[1, 1], [3, 4]])).toEqual([
-      [1, 4]
+      PositiveIntervals.union(
+        [
+          [1, 2],
+          [3, 4],
+        ],
+        [
+          [6, 7],
+          [19, 40],
+        ]
+      )
+    ).toEqual([
+      [1, 2],
+      [3, 4],
+      [6, 7],
+      [19, 40],
     ]);
+    expect(
+      PositiveIntervals.union(
+        [[1, 4]],
+        [
+          [1, 1],
+          [3, 4],
+        ]
+      )
+    ).toEqual([[1, 4]]);
     expect(PositiveIntervals.union([[3, 3]], [[4, 4]])).toEqual([
       [3, 3],
-      [4, 4]
+      [4, 4],
     ]);
   });
 });
@@ -70,34 +121,43 @@ describe("intersection", () => {
     expect(PositiveIntervals.intersection([[1, 2]], [[2, 3]])).toEqual([]);
     expect(PositiveIntervals.intersection([[2, 3]], [[1, 2]])).toEqual([]);
     expect(PositiveIntervals.intersection([[1, 10]], [[1, 10]])).toEqual([
-      [1, 10]
+      [1, 10],
     ]);
     expect(PositiveIntervals.intersection([[1, 10]], [[2, 8]])).toEqual([
-      [2, 8]
+      [2, 8],
     ]);
     expect(PositiveIntervals.intersection([[2, 8]], [[1, 10]])).toEqual([
-      [2, 8]
+      [2, 8],
     ]);
     expect(PositiveIntervals.intersection([[1, 10]], [[2, 12]])).toEqual([
-      [2, 10]
+      [2, 10],
     ]);
     expect(PositiveIntervals.intersection([[2, 12]], [[1, 10]])).toEqual([
-      [2, 10]
+      [2, 10],
     ]);
     expect(PositiveIntervals.intersection([[1, 10]], [[1, 8]])).toEqual([
-      [1, 8]
+      [1, 8],
     ]);
     expect(PositiveIntervals.intersection([[1, 8]], [[1, 10]])).toEqual([
-      [1, 8]
+      [1, 8],
     ]);
-    expect(PositiveIntervals.intersection([[1, 10]], [[1, 2], [6, 9]])).toEqual(
-      [[1, 2], [6, 9]]
-    );
-    expect(PositiveIntervals.intersection([[0, 2638]], [[1363, 2638]])).toEqual(
-      [[1363, 2638]]
-    );
+    expect(
+      PositiveIntervals.intersection(
+        [[1, 10]],
+        [
+          [1, 2],
+          [6, 9],
+        ]
+      )
+    ).toEqual([
+      [1, 2],
+      [6, 9],
+    ]);
+    expect(
+      PositiveIntervals.intersection([[0, 2638]], [[1363, 2638]])
+    ).toEqual([[1363, 2638]]);
     expect(PositiveIntervals.intersection([[1, 2]], [[1, 2]])).toEqual([
-      [1, 2]
+      [1, 2],
     ]);
   });
 });
@@ -110,32 +170,66 @@ describe("difference", () => {
   });
 
   test("simple", () => {
-    expect(PositiveIntervals.difference([[1, 2], [3, 4]], [])).toEqual([
+    expect(
+      PositiveIntervals.difference(
+        [
+          [1, 2],
+          [3, 4],
+        ],
+        []
+      )
+    ).toEqual([
       [1, 2],
-      [3, 4]
-    ]);
-    expect(PositiveIntervals.difference([[1, 2], [3, 10]], [[5, 10]])).toEqual([
-      [1, 2],
-      [3, 5]
-    ]);
-    expect(PositiveIntervals.difference([[1, 2], [3, 10]], [[0, 5]])).toEqual([
-      [5, 10]
+      [3, 4],
     ]);
     expect(
-      PositiveIntervals.difference([[0, 2638]], [[0, 1363], [2055, 2638]])
+      PositiveIntervals.difference(
+        [
+          [1, 2],
+          [3, 10],
+        ],
+        [[5, 10]]
+      )
+    ).toEqual([
+      [1, 2],
+      [3, 5],
+    ]);
+    expect(
+      PositiveIntervals.difference(
+        [
+          [1, 2],
+          [3, 10],
+        ],
+        [[0, 5]]
+      )
+    ).toEqual([[5, 10]]);
+    expect(
+      PositiveIntervals.difference(
+        [[0, 2638]],
+        [
+          [0, 1363],
+          [2055, 2638],
+        ]
+      )
     ).toEqual([[1363, 2055]]);
     expect(
-      PositiveIntervals.difference([[0, 1363], [2055, 2638]], [[0, 2638]])
+      PositiveIntervals.difference(
+        [
+          [0, 1363],
+          [2055, 2638],
+        ],
+        [[0, 2638]]
+      )
     ).toEqual([]);
     expect(PositiveIntervals.difference([[0, 10]], [[0, 1]])).toEqual([
-      [1, 10]
+      [1, 10],
     ]);
     expect(PositiveIntervals.difference([[0, 10]], [[1, 2]])).toEqual([
       [0, 1],
-      [2, 10]
+      [2, 10],
     ]);
     expect(PositiveIntervals.difference([[0, 10]], [[9, 10]])).toEqual([
-      [0, 9]
+      [0, 9],
     ]);
   });
 });

client/__tests__/util/typedCrossfilter/sort.test.js

@@ -2,9 +2,6 @@ import {
   sortArray,
   sortIndex,
   lowerBound,
-  upperBound,
-  lowerBoundIndirect,
-  upperBoundIndirect
 } from "../../../src/util/typedCrossfilter/sort";
 
 /*
@@ -40,7 +37,7 @@ describe("sortArray", () => {
       ["a", "b", "0", "1"],
       [0, "a", true, null, undefined, 3.1415],
       fillRand(new Array(1000)),
-      ["a", NaN, null, pInf]
+      ["a", NaN, null, pInf],
     ].map((val, idx) =>
       test(`JS vals ${idx}`, () => {
         expect(sortArray(val)).toMatchObject(val.sort());
@@ -49,7 +46,7 @@ describe("sortArray", () => {
   });
 
   describe("finite numbers", () => {
-    [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map(Type =>
+    [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map((Type) =>
       test(Type.name, () => {
         expect(sortArray(Type.from([6, 5, 4, 3, 2, 1, 0]))).toMatchObject(
           Type.from([0, 1, 2, 3, 4, 5, 6])
@@ -68,7 +65,7 @@ describe("sortArray", () => {
   });
 
   describe("non-finite numbers", () => {
-    test("inifinity", () => {
+    test("infinity", () => {
       expect(sortArray(new Float32Array([pInf, nInf, 0, 1, 2]))).toMatchObject(
         new Float32Array([nInf, 0, 1, 2, pInf])
       );
@@ -131,7 +128,7 @@ describe("sortArray", () => {
 
 describe("sortIndex", () => {
   describe("finite numbers", () => {
-    [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map(Type =>
+    [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map((Type) =>
       test(Type.name, () => {
         const source1 = Type.from([6, 5, 4, 3, 2, 1, 0]);
         const index1 = fillRange(new Uint32Array(source1.length));

client/__tests__/util/typedCrossfilter/util.test.js

@@ -1,6 +1,6 @@
 import {
   sliceByIndex,
-  makeSortIndex
+  makeSortIndex,
 } from "../../../src/util/typedCrossfilter/util";
 import { rangeFill as fillRange } from "../../../src/util/range";
 

client/configuration/babel/babel.dev.js

@@ -2,8 +2,15 @@ module.exports = {
   babelrc: false,
   cacheDirectory: true,
   presets: [
-    ["modern-browsers", { loose: true, modules: false }],
-    "@babel/preset-react"
+    [
+      "@babel/preset-env",
+      {
+        useBuiltIns: "entry",
+        corejs: 3,
+        modules: false,
+      },
+    ],
+    "@babel/preset-react",
   ],
   plugins: [
     "@babel/plugin-proposal-function-bind",
@@ -11,6 +18,6 @@ module.exports = {
     ["@babel/plugin-proposal-class-properties", { loose: true }],
     "@babel/plugin-proposal-export-namespace-from",
     "@babel/plugin-proposal-optional-chaining",
-    "@babel/plugin-proposal-nullish-coalescing-operator"
-  ]
+    "@babel/plugin-proposal-nullish-coalescing-operator",
+  ],
 };

client/configuration/babel/babel.prod.js

@@ -1,8 +1,15 @@
 module.exports = {
   babelrc: false,
   presets: [
-    ["modern-browsers", { loose: true, modules: false }],
-    "@babel/preset-react"
+    [
+      "@babel/preset-env",
+      {
+        useBuiltIns: "entry",
+        corejs: 3,
+        modules: false,
+      },
+    ],
+    "@babel/preset-react",
   ],
   plugins: [
     "@babel/plugin-proposal-function-bind",
@@ -12,6 +19,6 @@ module.exports = {
     "@babel/plugin-transform-react-constant-elements",
     "@babel/plugin-transform-runtime",
     "@babel/plugin-proposal-optional-chaining",
-    "@babel/plugin-proposal-nullish-coalescing-operator"
-  ]
+    "@babel/plugin-proposal-nullish-coalescing-operator",
+  ],
 };

client/configuration/eslint/eslint.js

@@ -1,18 +1,47 @@
 module.exports = {
   root: true,
   parser: "babel-eslint",
-  extends: ["airbnb", "prettier", "prettier/react"],
+  extends: [
+    "airbnb",
+    "plugin:eslint-comments/recommended",
+    "plugin:@blueprintjs/recommended",
+    "plugin:compat/recommended",
+    "plugin:prettier/recommended",
+    "prettier/react",
+  ],
+  settings: {
+    polyfills: [
+      "TextDecoder",
+      "TextEncoder",
+      "fetch",
+      "Request",
+      "Response",
+      "Headers",
+      "AbortController",
+    ],
+  },
   env: { browser: true, commonjs: true, es6: true },
-  globals: { expect: true },
+  globals: {
+    expect: true,
+    jest: true,
+    jestPuppeteer: true,
+    it: true,
+    page: true,
+    browser: true,
+    context: true,
+    beforeEach: true,
+  },
   parserOptions: {
     ecmaVersion: 2017,
     sourceType: "module",
     ecmaFeatures: {
       jsx: true,
-      generators: true
-    }
+      generators: true,
+    },
   },
   rules: {
+    "react/jsx-no-target-blank": "off",
+    "eslint-comments/require-description": ["error"],
     "no-magic-numbers": "off",
     "no-nested-ternary": "off",
     "func-style": "off",
@@ -21,13 +50,7 @@ module.exports = {
     "react/jsx-filename-extension": "off",
     "comma-dangle": "off",
     "no-underscore-dangle": "off",
-    quotes: ["error", "double"],
     "implicit-arrow-linebreak": "off",
-    "operator-linebreak": [
-      "error",
-      "after",
-      { overrides: { "?": "before", ":": "before" } }
-    ],
     "no-console": "off",
     "spaced-comment": ["error", "always", { exceptions: ["*"] }],
     "no-param-reassign": "off",
@@ -35,13 +58,26 @@ module.exports = {
     "react/prop-types": [0],
     "space-before-function-paren": "off",
     "function-paren-newline": "off",
-    "prefer-destructuring": ["error", { object: true, array: false }]
+    "prefer-destructuring": ["error", { object: true, array: false }],
+    "import/prefer-default-export": "off",
+    "no-restricted-syntax": [
+      "error",
+      "ForInStatement",
+      "LabeledStatement",
+      "WithStatement",
+    ],
+    "import/no-extraneous-dependencies": [
+      "error",
+      {
+        devDependencies: true,
+      },
+    ],
   },
   overrides: [
     {
       files: ["**/*.test.js"],
       env: {
-        jest: true // now **/*.test.js files' env has both es6 *and* jest
+        jest: true, // now **/*.test.js files' env has both es6 *and* jest
       },
       // Can't extend in overrides: https://github.com/eslint/eslint/issues/8813
       // "extends": ["plugin:jest/recommended"]
@@ -51,8 +87,8 @@ module.exports = {
         "jest/no-focused-tests": "error",
         "jest/no-identical-title": "error",
         "jest/prefer-to-have-length": "warn",
-        "jest/valid-expect": "error"
-      }
-    }
-  ]
+        "jest/valid-expect": "error",
+      },
+    },
+  ],
 };

client/configuration/lint-staged/lint-staged.config.js

@@ -0,0 +1,3 @@
+module.exports = {
+  "*.js": "eslint --fix",
+};

client/configuration/webpack/cspHashPlugin.js

@@ -0,0 +1,53 @@
+/* eslint-disable import/no-extraneous-dependencies -- this file is a devDependency*/
+const cheerio = require("cheerio");
+const crypto = require("crypto");
+const HtmlWebpackPlugin = require("html-webpack-plugin");
+
+const digest = (str) => {
+  const hash = crypto.createHash("sha256").update(str, "utf8").digest("base64");
+  return `sha256-${hash}`;
+};
+class CspHashPlugin {
+  constructor(opts) {
+    this.opts = { ...opts };
+  }
+
+  apply(compiler) {
+    compiler.hooks.compilation.tap("CspHashPlugin", (compilation) => {
+      HtmlWebpackPlugin.getHooks(compilation).beforeEmit.tapAsync(
+        "CspHashPlugin",
+        (data, cb) => {
+          const { filename } = this.opts;
+
+          const $ = cheerio.load(data.html, { decodeEntities: false });
+
+          if (filename) {
+            const results = {};
+            results["script-hashes"] = $("script:not([src]):not([no-csp-hash])")
+              .map((i, elmt) => digest($(elmt).html()))
+              .get();
+
+            const json = JSON.stringify(results);
+            compilation.assets[filename] = {
+              source: () => json,
+              size: () => json.length,
+            };
+          }
+
+          // Remove no-csp-hash attributes. Cheerio does not parse Jinja templates
+          // correctly, so we brute force this with a regular expression.
+          data.html = data.html.replace(
+            /(<script .*)no-csp-hash(.*>)/,
+            (match, p1, p2) => [p1, p2].join("")
+          );
+
+          // Tell webpack to move on
+          cb(null, data);
+        }
+      );
+    });
+  }
+}
+
+module.exports = CspHashPlugin;
+/* eslint-enable import/no-extraneous-dependencies -- enable*/

client/configuration/webpack/obsoleteHTMLTemplate.html

@@ -0,0 +1,82 @@
+<script>
+  /** Reflect ANY changes to this script in the script hash in `server/eb/app.py` **/
+  var root = document.getElementById("root");
+  root.remove();
+  var portals = document.getElementsByClassName("bp3-portal");
+  for (var i = 0; i < portals.length; i += 1) {
+    portals[i].remove();
+  }
+</script>
+<div
+  style="
+    display: flex;
+    flex-direction: column;
+    width: 100vw;
+    height: 100vh;
+    text-align: center;
+    justify-content: center;
+    align-items: center;
+    background: #8080801a;
+    font-family: 'Roboto Condensed, sans serif';
+  "
+>
+  <img
+    src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/cellxgene-logo.png"
+    style="width: 320px;"
+  />
+  <div
+    style="
+      margin-top: 16px;
+      background: white;
+      width: 40vw;
+      border-radius: 4px;
+      padding: 24px 64px;
+      -webkit-box-shadow: 0px 0px 3px 2px rgba(0, 0, 0, 0.38);
+      -moz-box-shadow: 0px 0px 3px 2px rgba(0, 0, 0, 0.38);
+      box-shadow: 0px 0px 3px 2px rgba(0, 0, 0, 0.38);
+      max-width: 550px;
+    "
+  >
+    <div style="margin-bottom: 0; font-weight: bolder; font-size: 1.2em;">
+      Unsupported Browser
+    </div>
+    <div style="margin-top: 0;">
+      cellxgene is currently supported on the following browsers
+    </div>
+    <div
+      style="display: flex; justify-content: space-around; margin-top: 16px;"
+    >
+      <a
+        href="https://www.google.com/chrome/?hl=en%22"
+        aria-label="Download Google Chrome"
+      >
+        <img
+          src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/chrome.png"
+          style="width: 80px; height: 80px;"
+        />
+        <div>Chrome &gt; 60</div>
+      </a>
+      <a href="https://www.apple.com/safari/" aria-label="Download Safari">
+        <img
+          src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/safari.png"
+          style="width: 80px; height: 80px;"
+        />
+        <div>Safari ≥ 10.1</div>
+      </a>
+      <a href="https://www.mozilla.com/firefox/" aria-label="Download Firefox">
+        <img
+          src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/firefox.png"
+          style="width: 80px; height: 80px;"
+        />
+        <div>Firefox ≥ 60</div>
+      </a>
+      <a href="//www.microsoft.com/edge" aria-label="Download Edge">
+        <img
+          src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/edge.png"
+          style="width: 80px; height: 80px;"
+        />
+        <div>Edge ≥ 79</div>
+      </a>
+    </div>
+  </div>
+</div>

client/configuration/webpack/webpack.config.dev.js

@@ -1,93 +1,80 @@
-// jshint esversion: 6
 const path = require("path");
 const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
+const FaviconsWebpackPlugin = require("favicons-webpack-plugin");
+const ScriptExtHtmlWebpackPlugin = require("script-ext-html-webpack-plugin");
+const MiniCssExtractPlugin = require("mini-css-extract-plugin");
+
+const { merge } = require("webpack-merge");
+
+const sharedConfig = require("./webpack.config.shared");
+const babelOptions = require("../babel/babel.dev");
 
-const src = path.resolve("src");
 const fonts = path.resolve("src/fonts");
 const nodeModules = path.resolve("node_modules");
 
-const babelOptions = require("../babel/babel.dev");
-
-module.exports = {
+const devConfig = {
   mode: "development",
   devtool: "eval",
-  entry: ["./src/index"],
   output: {
-    path: path.resolve("build"),
     pathinfo: true,
     filename: "static/js/bundle.js",
-    publicPath: "/"
   },
   module: {
     rules: [
       {
-        test: /\.js$/,
-        include: src,
+        test: /\.jsx?$/,
         loader: "babel-loader",
-        options: babelOptions
+        options: babelOptions,
       },
       {
-        test: /\.css$/,
-        include: src,
-        exclude: [path.resolve(src, "index.css")],
-        loader: [
-          {
-            loader: "style-loader"
-          },
-          {
-            loader: "css-loader",
-            options: {
-              modules: {
-                localIdentName: "[name]__[local]___[hash:base64:5]"
-              }
-            }
-          }
-        ]
-      },
-      {
-        test: /index\.css$/,
-        include: [path.resolve(src, "index.css")],
-        loader: [
-          {
-            loader: "style-loader"
-          },
-          {
-            loader: "css-loader"
-          }
-        ]
-      },
-
-      { test: /\.json$/, include: [src, nodeModules], loader: "json-loader" },
-      {
-        test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2)(\?.*)?$/,
-        include: nodeModules,
+        test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i,
         loader: "file-loader",
-        query: { name: "static/media/[name].[ext]" }
+        include: [nodeModules, fonts],
+        query: {
+          name: "static/assets/[name].[ext]",
+          // (thuang): This is needed to make sure @font url path is '/static/assets/'
+          publicPath: "/",
+        },
       },
-      {
-        test: /\.(woff|woff2|eot|ttf|otf)$/,
-        include: fonts,
-        loader: "file-loader",
-        query: { name: "static/fonts/[name].[ext]" }
-      },
-      {
-        test: /\.(mp4|webm)(\?.*)?$/,
-        include: [src, nodeModules],
-        loader: "url-loader",
-        query: { limit: 10000, name: "static/media/[name].[ext]" }
-      }
-    ]
+    ],
   },
   plugins: [
     new HtmlWebpackPlugin({
       inject: true,
       template: path.resolve("index.html"),
-      favicon: path.resolve("favicon.png")
+    }),
+    new FaviconsWebpackPlugin({
+      logo: "./favicon.png",
+      prefix: "static/img/",
+      favicons: {
+        icons: {
+          android: false,
+          appleIcon: false,
+          appleStartup: false,
+          coast: false,
+          firefox: false,
+          windows: false,
+          yandex: false,
+        },
+      },
+    }),
+    new MiniCssExtractPlugin({
+      filename: "static/[name].css",
     }),
     new webpack.NoEmitOnErrorsPlugin(),
     new webpack.DefinePlugin({
-      __REACT_DEVTOOLS_GLOBAL_HOOK__: "({ isDisabled: true })"
-    })
-  ]
+      __REACT_DEVTOOLS_GLOBAL_HOOK__: "({ isDisabled: true })",
+    }),
+    new webpack.DefinePlugin({
+      "process.env.CXG_SERVER_PORT": JSON.stringify(
+        process.env.CXG_SERVER_PORT
+      ),
+    }),
+    new ScriptExtHtmlWebpackPlugin({
+      async: "obsolete",
+    }),
+  ],
 };
+
+module.exports = merge(sharedConfig, devConfig);

client/configuration/webpack/webpack.config.prod.js

@@ -1,124 +1,96 @@
-// jshint esversion: 6
 const path = require("path");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
+const { CleanWebpackPlugin } = require("clean-webpack-plugin");
+const TerserJSPlugin = require("terser-webpack-plugin");
+const CleanCss = require("clean-css");
+const OptimizeCSSAssetsPlugin = require("optimize-css-assets-webpack-plugin");
+const FaviconsWebpackPlugin = require("favicons-webpack-plugin");
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
-const SWPrecacheWebpackPlugin = require("sw-precache-webpack-plugin");
-const HtmlWebpackInlineSourcePlugin = require("html-webpack-inline-source-plugin");
 
-const src = path.resolve("src");
-const fonts = path.resolve("src/fonts");
-const nodeModules = path.resolve("node_modules");
+const { merge } = require("webpack-merge");
 
 const babelOptions = require("../babel/babel.prod");
 
-const publicPath = "";
+const CspHashPlugin = require("./cspHashPlugin");
+const sharedConfig = require("./webpack.config.shared");
 
-module.exports = {
+const fonts = path.resolve("src/fonts");
+const nodeModules = path.resolve("node_modules");
+
+const prodConfig = {
   mode: "production",
   bail: true,
   cache: false,
-  devtool: "cheap-source-map",
-  entry: ["./src/index.js"],
   output: {
-    path: path.resolve("build"),
-    filename: "static/js/[name].[chunkhash:8].js",
-    publicPath
+    filename: "static/[name]-[contenthash].js",
   },
+  optimization: {
+    minimize: true,
+    minimizer: [
+      new TerserJSPlugin({}),
+      new OptimizeCSSAssetsPlugin({
+        cssProcessor: CleanCss,
+      }),
+    ],
+  },
+  devtool: "source-map",
   module: {
     rules: [
       {
-        test: /\.js$/,
-        include: src,
+        test: /\.jsx?$/,
         loader: "babel-loader",
-        options: babelOptions
+        options: babelOptions,
       },
       {
-        test: /\.css$/,
-        include: src,
-        exclude: [path.resolve(src, "index.css")],
-        use: [
-          MiniCssExtractPlugin.loader,
-          {
-            loader: "css-loader",
-            options: {
-              modules: {
-                localIdentName: "[name]__[local]___[hash:base64:5]"
-              },
-              importLoaders: 1
-            }
-          }
-        ]
-      },
-      {
-        test: /index\.css$/,
-        include: [path.resolve(src, "index.css")],
-        use: [
-          MiniCssExtractPlugin.loader,
-          {
-            loader: "css-loader",
-            options: {
-              importLoaders: 1
-            }
-          }
-        ]
-      },
-      {
-        test: /\.json$/,
-        include: [src, nodeModules],
-        loader: "json-loader",
-        exclude: /manifest.json$/
-      },
-      {
-        test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2)(\?.*)?$/,
-        include: nodeModules,
+        test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i,
         loader: "file-loader",
-        query: { name: "static/media/[name].[ext]" }
+        include: [nodeModules, fonts],
+        query: {
+          name: "static/assets/[name]-[contenthash].[ext]",
+          // (thuang): This is needed to make sure @font url path is '../static/assets/'
+          publicPath: "static/",
+        },
       },
-      {
-        test: /\.(woff|woff2|eot|ttf|otf)$/,
-        include: fonts,
-        loader: "file-loader",
-        query: { name: "static/fonts/[name].[ext]" }
-      },
-      {
-        test: /\.(mp4|webm)(\?.*)?$/,
-        include: [src, nodeModules],
-        loader: "url-loader",
-        query: { limit: 10000, name: "static/media/[name].[hash:8].[ext]" }
-      }
-    ]
+    ],
   },
   plugins: [
     new HtmlWebpackPlugin({
-      inject: "body",
       filename: "index.html",
       template: path.resolve("index_template.html"),
-      favicon: path.resolve("favicon.png"),
-      inlineSource: ".(js|css)$",
-      minify: {
-        removeComments: true,
-        collapseWhitespace: true,
-        removeRedundantAttributes: true,
-        useShortDoctype: true,
-        removeEmptyAttributes: true,
-        removeStyleLinkTypeAttributes: true,
-        keepClosingSlash: true,
-        minifyJS: true,
-        minifyCSS: true,
-        minifyURLs: true
-      }
+      decodeEntities: false,
+      minify: false,
+    }),
+    new CleanWebpackPlugin({
+      verbose: true,
+      protectWebpackAssets: false,
+      cleanAfterEveryBuildPatterns: ["main.js", "main.css"],
+    }),
+    new FaviconsWebpackPlugin({
+      logo: "./favicon.png",
+      prefix: "static/assets/",
+      favicons: {
+        icons: {
+          android: false,
+          appleIcon: false,
+          appleStartup: false,
+          coast: false,
+          firefox: false,
+          windows: false,
+          yandex: false,
+        },
+      },
     }),
-    new HtmlWebpackInlineSourcePlugin(),
     new MiniCssExtractPlugin({
-      filename: "static/css/[name].[contenthash:8].css"
+      filename: "static/[name]-[contenthash].css",
+    }),
+    new CspHashPlugin({
+      filename: "csp-hashes.json",
     }),
-    new SWPrecacheWebpackPlugin({
-      cacheId: "cellxgene",
-      filename: "service-worker.js"
-    })
   ],
   performance: {
     maxEntrypointSize: 2000000,
-    maxAssetSize: 2000000
-  }
+    maxAssetSize: 2000000,
+  },
 };
+
+module.exports = merge(sharedConfig, prodConfig);

client/configuration/webpack/webpack.config.shared.js

@@ -0,0 +1,74 @@
+const path = require("path");
+const fs = require("fs");
+const MiniCssExtractPlugin = require("mini-css-extract-plugin");
+const ObsoleteWebpackPlugin = require("obsolete-webpack-plugin");
+const ScriptExtHtmlWebpackPlugin = require("script-ext-html-webpack-plugin");
+
+const src = path.resolve("src");
+const nodeModules = path.resolve("node_modules");
+
+const publicPath = "";
+
+const rawObsoleteHTMLTemplate = fs.readFileSync(
+  `${__dirname}/obsoleteHTMLTemplate.html`,
+  "utf8"
+);
+
+const obsoleteHTMLTemplate = rawObsoleteHTMLTemplate.replace(/'/g, '"');
+
+module.exports = {
+  entry: [
+    "core-js",
+    "regenerator-runtime/runtime",
+    "fastestsmallesttextencoderdecoder",
+    "whatwg-fetch",
+    "abort-controller/polyfill",
+    "./src/index",
+  ],
+  output: {
+    path: path.resolve("build"),
+    publicPath,
+  },
+  module: {
+    rules: [
+      {
+        test: /\.css$/,
+        include: src,
+        exclude: [path.resolve(src, "index.css")],
+        use: [
+          MiniCssExtractPlugin.loader,
+          {
+            loader: "css-loader",
+            options: {
+              modules: {
+                localIdentName: "[name]__[local]___[hash:base64:5]",
+              },
+              importLoaders: 1,
+            },
+          },
+        ],
+      },
+      {
+        test: /index\.css$/,
+        include: [path.resolve(src, "index.css")],
+        use: [MiniCssExtractPlugin.loader, "css-loader"],
+      },
+      {
+        test: /\.json$/,
+        include: [src, nodeModules],
+        loader: "json-loader",
+        exclude: /manifest.json$/,
+      },
+    ],
+  },
+  plugins: [
+    new ObsoleteWebpackPlugin({
+      name: "obsolete",
+      template: obsoleteHTMLTemplate,
+      promptOnNonTargetBrowser: false,
+    }),
+    new ScriptExtHtmlWebpackPlugin({
+      async: "obsolete",
+    }),
+  ],
+};

client/index.html

@@ -1,12 +1,27 @@
 <!DOCTYPE html>
 <html lang="en">
   <head>
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
     <title>cell&times;gene</title>
     <style>
-      html, body, p, h1, h2, h3, h4, h5, h6, span, button, input, label, text, div {
-        font-family: 'Roboto Condensed','Helvetica Neue','Helvetica','Arial',sans-serif;
+      html,
+      body,
+      p,
+      h1,
+      h2,
+      h3,
+      h4,
+      h5,
+      h6,
+      span,
+      button,
+      input,
+      label,
+      text,
+      div {
+        font-family: "Roboto Condensed", "Helvetica Neue", "Helvetica", "Arial",
+          sans-serif;
         font-size: 14px;
       }
       body {
@@ -20,9 +35,12 @@
     </style>
   </head>
   <body>
-    <noscript>If you're seeing this message, that means <strong>JavaScript has been disabled on your browser</strong>, please <strong>enable JS</strong> to make this app work.</noscript>
+    <noscript
+      >If you're seeing this message, that means
+      <strong>JavaScript has been disabled on your browser</strong>, please
+      <strong>enable JS</strong> to make this app work.</noscript
+    >
 
     <div id="root"></div>
-
   </body>
 </html>

client/index_template.html

@@ -1,41 +1,65 @@
 <!DOCTYPE html>
 <html lang="en">
-<head>
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
     <title>cell&times;gene</title>
     <style>
-        html, body, p, h1, h2, h3, h4, h5, h6, span, button, input, label, text, div {
-            font-family: 'Roboto Condensed', 'Helvetica Neue', 'Helvetica', 'Arial', sans-serif;
-            font-size: 14px;
-        }
+      html,
+      body,
+      p,
+      h1,
+      h2,
+      h3,
+      h4,
+      h5,
+      h6,
+      span,
+      button,
+      input,
+      label,
+      text,
+      div {
+        font-family: "Roboto Condensed", "Helvetica Neue", "Helvetica", "Arial",
+          sans-serif;
+        font-size: 14px;
+      }
 
-        body {
-            margin: 0;
-            padding: 0;
-        }
+      body {
+        margin: 0;
+        padding: 0;
+      }
 
-        * {
-            box-sizing: border-box;
-        }
+      * {
+        box-sizing: border-box;
+      }
     </style>
+  </head>
+  <body>
+    <script type="text/javascript">
+      window.CELLXGENE = {};
+      window.CELLXGENE.API = {
+        prefix: `${location.origin}${location.pathname}api/`,
+        version: "v0.2/",
+      };
+    </script>
+    <noscript
+      >If you're seeing this message, that means
+      <strong>JavaScript has been disabled on your browser</strong>, please
+      <strong>enable JS</strong> to make this app work.
+    </noscript>
 
-</head>
-<body>
-<script type="text/javascript">
-  window.CELLXGENE = {};
-  window.CELLXGENE.API = {
-    prefix: window.location.href + "api/",
-    version: "v0.2/"
-  };
-</script>
-<noscript>If you're seeing this message, that means <strong>JavaScript has been disabled on your browser</strong>,
-    please <strong>enable JS</strong> to make this app work.
-</noscript>
+    <div id="root"></div>
+    {% for script in SCRIPTS -%}
+    <script type="text/javascript"
+      {{ ('integrity="%s"' % script.integrity) | safe if script.integrity }}
+      {{ ('crossorigin="%s"' % script.crossorigin) | safe if script.crossorigin }}
+      src="{{script.src | safe}}"
+    ></script>
+    {%- endfor %}
 
-<div id="root"></div>
-{% for script in SCRIPTS %}
-<script type="text/javascript" src="{{script | safe}}"></script>
-{% endfor %}
-</body>
+    {% for ils in INLINE_SCRIPTS %}
+    <script type="text/javascript" no-csp-hash>{% include ils %}</script>
+    {% endfor %}
+  </body>
 </html>

client/jest-puppeteer.config.js

@@ -0,0 +1,46 @@
+/**
+ * `client/jest-puppeteer.config.js` is for configuring Puppeteer's launch config options
+ * `client/__tests__/e2e/puppeteer.setup.js` is for configuring `jest`, `browser`,
+ * and `page` objects
+ */
+
+const ENV_DEFAULT = require("../environment.default.json");
+
+const jestEnv = process.env.JEST_ENV || ENV_DEFAULT.JEST_ENV;
+const isHeadful =
+  process.env.HEADFUL === "true" || process.env.HEADLESS === "false";
+
+const DEFAULT_LAUNCH_CONFIG = {
+  headless: !isHeadful,
+  args: ["--ignore-certificate-errors", "--ignore-ssl-errors"],
+  ignoreHTTPSErrors: true,
+  defaultViewport: {
+    width: 1280,
+    height: 960,
+  },
+};
+
+const LAUNCH_CONFIG_BY_ENV = {
+  [ENV_DEFAULT.DEBUG]: {
+    ...DEFAULT_LAUNCH_CONFIG,
+    headless: false,
+    slowMo: 100,
+    devtools: true,
+    defaultViewport: {
+      width: DEFAULT_LAUNCH_CONFIG.defaultViewport.width,
+      height: DEFAULT_LAUNCH_CONFIG.defaultViewport.height + 560,
+    },
+  },
+  [ENV_DEFAULT.DEV]: {
+    ...DEFAULT_LAUNCH_CONFIG,
+    headless: false,
+    slowMo: 5,
+  },
+};
+
+const launchConfig = LAUNCH_CONFIG_BY_ENV[jestEnv] || DEFAULT_LAUNCH_CONFIG;
+
+module.exports = {
+  browserContext: "incognito",
+  launch: launchConfig,
+};

client/package.json

@@ -1,24 +1,20 @@
 {
   "name": "cellxgene",
-  "version": "0.14.0",
+  "version": "0.16.6",
   "license": "MIT",
   "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
   "repository": "https://github.com/chanzuckerberg/cellxgene",
   "scripts": {
-    "backend-dev": "python3.6 -m venv cellxgene && source cellxgene/bin/activate && yes | pip uninstall cellxgene || true && pip install -e .. && cellxgene launch ",
-    "backend-dev-anno-ontology": "python3.6 -m venv cellxgene && source cellxgene/bin/activate && yes | pip uninstall cellxgene || true && pip install -e .. && cellxgene launch --experimental-annotations --experimental-annotations-ontology",
-    "backend-dev-anno": "python3.6 -m venv cellxgene && source cellxgene/bin/activate && yes | pip uninstall cellxgene || true && pip install -e .. && cellxgene launch --experimental-annotations",
-    "build": "npm run clean && webpack --config configuration/webpack/webpack.config.prod.js",
+    "build": "npm run clean && webpack --config",
     "clean": "rimraf build",
-    "dev": "npm run clean && webpack --config configuration/webpack/webpack.config.dev.js",
-    "e2e": "node node_modules/jest/bin/jest.js --verbose false --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
-    "fmt": "eslint --fix src",
-    "lint": "eslint src",
-    "smoke-test": "start-server-and-test start-server-for-test :5000 e2e",
-    "start": "node server/development.js",
-    "start-server-for-test": "cellxgene launch -p 5000 ../example-dataset/pbmc3k.h5ad",
-    "test": "node node_modules/jest/bin/jest.js",
-    "unit-test": "node node_modules/jest/bin/jest.js --testPathIgnorePatterns e2e"
+    "dev": "npm run build -- configuration/webpack/webpack.config.dev.js",
+    "e2e": "jest --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
+    "e2e-annotations": "jest --config __tests__/e2e/e2eJestConfig.json e2e/e2eAnnotations.test.js",
+    "e2e-prod": "CXG_URL_BASE='https://cellxgene.cziscience.com/d/pbmc3k.cxg/' jest --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
+    "fmt": "eslint --fix src __tests__",
+    "lint": "eslint --fix src __tests__",
+    "prod": "npm run build -- configuration/webpack/webpack.config.prod.js",
+    "test": "jest --testPathIgnorePatterns e2e"
   },
   "engineStrict": true,
   "engines": {
@@ -33,90 +29,124 @@
   "resolutions": {
     "eslint-scope": "3.7.1"
   },
+  "browserslist": [
+    "Chrome > 60",
+    "Safari >= 10.1",
+    "iOS >= 10.3",
+    "Firefox >= 60",
+    "Edge >= 79",
+    "not Explorer > 0"
+  ],
   "dependencies": {
-    "@blueprintjs/core": "^3.22.3",
-    "@blueprintjs/icons": "^3.13.0",
-    "@blueprintjs/select": "^3.11.2",
+    "@blueprintjs/core": "^3.30.0",
+    "@blueprintjs/icons": "^3.19.0",
+    "@blueprintjs/select": "^3.13.5",
+    "abort-controller": "^3.0.0",
+    "core-js": "^3.6.5",
     "d3": "^4.10.0",
     "d3-scale-chromatic": "^1.5.0",
+    "fastestsmallesttextencoderdecoder": "^1.0.22",
     "flatbuffers": "^1.11.0",
     "fuzzysort": "^1.1.4",
-    "gl-mat4": "^1.1.4",
-    "gl-matrix": "^3.1.0",
+    "gl-mat4": "^1.2.0",
+    "gl-matrix": "^3.3.0",
     "gl-vec3": "^1.1.3",
     "is-number": "^7.0.0",
-    "lodash": "^4.17.15",
+    "lodash": "^4.17.20",
     "memoize-one": "^5.1.1",
-    "react": "^16.12.0",
-    "react-autocomplete": "^1.7.2",
-    "react-dom": "^16.12.0",
-    "react-flip-toolkit": "7.0.6",
+    "react": "^16.13.1",
+    "react-async": "^10.0.1",
+    "react-dom": "^16.13.1",
+    "react-flip-toolkit": "^7.0.12",
     "react-helmet": "^5.2.1",
-    "react-icons": "^3.8.0",
-    "react-redux": "^7.1.3",
+    "react-icons": "^3.10.0",
+    "react-redux": "^7.2.0",
     "redux": "^4.0.5",
-    "redux-thunk": "^2.2.0",
-    "regl": "^1.3.13"
+    "redux-thunk": "^2.3.0",
+    "regenerator-runtime": "^0.13.7",
+    "regl": "^1.6.1",
+    "script-ext-html-webpack-plugin": "^2.1.4",
+    "tinyqueue": "^2.0.3",
+    "webpack-merge": "^5.0.9",
+    "whatwg-fetch": "^3.2.0"
   },
   "devDependencies": {
-    "@babel/core": "^7.7.7",
-    "@babel/plugin-proposal-class-properties": "^7.7.4",
-    "@babel/plugin-proposal-decorators": "^7.7.4",
-    "@babel/plugin-proposal-export-namespace-from": "^7.7.4",
-    "@babel/plugin-proposal-function-bind": "^7.7.4",
-    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.7.4",
-    "@babel/plugin-proposal-optional-chaining": "^7.7.5",
-    "@babel/plugin-transform-react-constant-elements": "^7.7.4",
-    "@babel/plugin-transform-runtime": "^7.7.6",
-    "@babel/preset-env": "^7.7.7",
-    "@babel/preset-react": "^7.7.4",
-    "@babel/register": "^7.7.7",
-    "@babel/runtime": "^7.7.7",
-    "babel-eslint": "^10.0.3",
-    "babel-jest": "^24.9.0",
-    "babel-loader": "^8.0.6",
-    "babel-preset-modern-browsers": "^14.1.1",
-    "chalk": "^3.0.0",
+    "@babel/core": "^7.10.5",
+    "@babel/plugin-proposal-class-properties": "^7.10.4",
+    "@babel/plugin-proposal-decorators": "^7.10.5",
+    "@babel/plugin-proposal-export-namespace-from": "^7.10.4",
+    "@babel/plugin-proposal-function-bind": "^7.10.5",
+    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.10.4",
+    "@babel/plugin-proposal-optional-chaining": "^7.10.4",
+    "@babel/plugin-transform-react-constant-elements": "^7.10.4",
+    "@babel/plugin-transform-runtime": "^7.10.5",
+    "@babel/preset-env": "^7.10.4",
+    "@babel/preset-react": "^7.10.4",
+    "@babel/register": "^7.10.5",
+    "@babel/runtime": "^7.10.5",
+    "@blueprintjs/eslint-plugin": "^0.3.0",
+    "@sentry/webpack-plugin": "^1.12.0",
+    "babel-eslint": "^10.1.0",
+    "babel-jest": "^26.1.0",
+    "babel-loader": "^8.1.0",
+    "babel-preset-modern-browsers": "^14.2.1",
+    "chalk": "^4.1.0",
+    "cheerio": "^1.0.0-rc.3",
+    "clean-css": "^4.2.3",
+    "clean-webpack-plugin": "^3.0.0",
+    "codecov": "^3.7.1",
     "connect-history-api-fallback": "^1.6.0",
-    "copy-webpack-plugin": "^5.1.1",
-    "css-loader": "^3.4.1",
-    "eslint": "^6.8.0",
-    "eslint-config-airbnb": "^18.0.1",
-    "eslint-config-prettier": "^6.9.0",
-    "eslint-loader": "^3.0.3",
+    "css-loader": "^3.6.0",
+    "eslint": "^7.4.0",
+    "eslint-config-airbnb": "^18.2.0",
+    "eslint-config-prettier": "^6.11.0",
+    "eslint-loader": "^3.0.4",
+    "eslint-plugin-compat": "^3.8.0",
+    "eslint-plugin-eslint-comments": "^3.2.0",
     "eslint-plugin-filenames": "^1.3.2",
-    "eslint-plugin-import": "^2.19.1",
-    "eslint-plugin-jest": "^23.3.0",
-    "eslint-plugin-jsx-a11y": "^6.2.3",
-    "eslint-plugin-react": "^7.17.0",
-    "eslint-plugin-react-hooks": "^2.3.0",
+    "eslint-plugin-import": "^2.22.0",
+    "eslint-plugin-jest": "^23.18.0",
+    "eslint-plugin-jsx-a11y": "^6.3.1",
+    "eslint-plugin-prettier": "^3.1.4",
+    "eslint-plugin-react": "^7.20.3",
+    "eslint-plugin-react-hooks": "^4.0.8",
+    "expect-puppeteer": "^4.4.0",
     "express": "^4.17.1",
-    "file-loader": "^5.0.2",
-    "html-webpack-inline-source-plugin": "0.0.10",
-    "html-webpack-plugin": "^3.2.0",
-    "jest": "^24.9.0",
+    "favicons-webpack-plugin": "^3.0.1",
+    "file-loader": "^6.0.0",
+    "html-webpack-plugin": "^4.3.0",
+    "husky": "^4.2.5",
+    "jest": "^26.1.0",
+    "jest-circus": "^26.1.0",
+    "jest-environment-puppeteer": "^4.4.0",
+    "jest-fetch-mock": "^3.0.3",
     "jest-puppeteer": "^4.4.0",
-    "json-loader": "^0.5.4",
+    "json-loader": "^0.5.7",
+    "lint-staged": "^10.2.11",
     "mini-css-extract-plugin": "^0.9.0",
-    "puppeteer": "^2.0.0",
-    "rimraf": "^3.0.0",
-    "serve-favicon": "^2.3.0",
-    "start-server-and-test": "^1.10.6",
-    "style-loader": "^1.1.2",
-    "sw-precache-webpack-plugin": "^0.11.5",
-    "url-loader": "^3.0.0",
-    "webpack": "^4.41.5",
-    "webpack-cli": "^3.3.10",
+    "obsolete-webpack-plugin": "^0.5.6",
+    "optimize-css-assets-webpack-plugin": "^5.0.3",
+    "prettier": "^2.0.5",
+    "puppeteer": "^3.3.0",
+    "rimraf": "^3.0.2",
+    "serve-favicon": "^2.5.0",
+    "style-loader": "^1.2.1",
+    "sw-precache-webpack-plugin": "^1.0.0",
+    "terser-webpack-plugin": "^3.0.7",
+    "url-loader": "^4.1.0",
+    "webpack": "^4.43.0",
+    "webpack-cli": "^3.3.12",
     "webpack-dev-middleware": "^3.7.2"
   },
   "jest": {
     "testMatch": [
       "**/__tests__/**/?(*.)(spec|test).js?(x)"
     ],
-    "testURL": "http://localhost/",
     "setupFiles": [
       "./__tests__/setupMissingGlobals.js"
-    ]
+    ],
+    "coverageDirectory": "./coverage/",
+    "collectCoverage": true
   },
   "babel": {
     "env": {
@@ -147,5 +177,10 @@
         ]
       }
     }
+  },
+  "husky": {
+    "hooks": {
+      "pre-commit": "lint-staged --config \"./configuration/lint-staged/lint-staged.config.js\""
+    }
   }
 }

client/server/development.js

@@ -1,49 +1,62 @@
-/* eslint-disable */
-// jshint esversion: 6
-var path = require("path");
-var historyApiFallback = require("connect-history-api-fallback");
-var chalk = require("chalk");
-var express = require("express");
-var favicon = require("serve-favicon");
-var webpack = require("webpack");
-var config = require("../configuration/webpack/webpack.config.dev");
-var utils = require("./utils");
+const chalk = require("chalk");
+const express = require("express");
+const favicon = require("serve-favicon");
+const webpack = require("webpack");
+const devMiddleware = require("webpack-dev-middleware");
+const config = require("../configuration/webpack/webpack.config.dev");
+const utils = require("./utils");
 
 process.env.NODE_ENV = "development";
 
-var PORT = process.env.PORT || 3000;
+const CLIENT_PORT = process.env.CXG_CLIENT_PORT;
+const { CXG_SERVER_PORT } = process.env;
+
+const API = {
+  prefix: `http://localhost:${CXG_SERVER_PORT}/`,
+};
 
 // Set up compiler
-var compiler = webpack(config);
+const compiler = webpack(config);
 
-compiler.plugin("invalid", () => {
+compiler.hooks.invalid.tap("invalid", () => {
   utils.clearConsole();
   console.log("Compiling...");
 });
 
-compiler.plugin("done", stats => {
-  utils.formatStats(stats, PORT);
+compiler.hooks.done.tap("done", (stats) => {
+  utils.formatStats(stats, CLIENT_PORT);
 });
 
 // Launch server
-var app = express();
-
-app.use(historyApiFallback({ verbose: false }));
+const app = express();
 
 app.use(
-  require("webpack-dev-middleware")(compiler, {
+  devMiddleware(compiler, {
     logLevel: "warn",
-    publicPath: config.output.publicPath
+    publicPath: config.output.publicPath,
+    index: true,
   })
 );
 
 app.use(favicon("./favicon.png"));
 
-app.get("*", (req, res) => {
-  res.sendFile(path.resolve("index.html"));
+app.get("/login", async (req, res) => {
+  try {
+    res.redirect(`${API.prefix}login?dataset=http://localhost:${CLIENT_PORT}`);
+  } catch (err) {
+    console.error(err);
+  }
 });
 
-app.listen(PORT, err => {
+app.get("/logout", async (req, res) => {
+  try {
+    res.redirect(`${API.prefix}logout?dataset=http://localhost:${CLIENT_PORT}`);
+  } catch (err) {
+    console.error(err);
+  }
+});
+
+app.listen(CLIENT_PORT, (err) => {
   if (err) {
     console.log(err);
     return;

client/server/utils.js

@@ -36,10 +36,10 @@ var formatStats = (stats, port) => {
 
   var json = stats.toJson();
   var formattedErrors = json.errors.map(
-    message => "Error in " + formatMessage(message)
+    (message) => "Error in " + formatMessage(message)
   );
   var formattedWarnings = json.warnings.map(
-    message => "Warning in " + formatMessage(message)
+    (message) => "Warning in " + formatMessage(message)
   );
 
   if (hasErrors) {
@@ -48,7 +48,7 @@ var formatStats = (stats, port) => {
     if (formattedErrors.some(isLikelyASyntaxError)) {
       formattedErrors = formattedErrors.filter(isLikelyASyntaxError);
     }
-    formattedErrors.forEach(message => {
+    formattedErrors.forEach((message) => {
       console.log(message);
       console.log();
     });
@@ -58,7 +58,7 @@ var formatStats = (stats, port) => {
   if (hasWarnings) {
     console.log(chalk.yellow("Compiled with warnings."));
     console.log();
-    formattedWarnings.forEach(message => {
+    formattedWarnings.forEach((message) => {
       console.log(message);
       console.log();
     });

client/src/actions/annotation.js

@@ -0,0 +1,387 @@
+/*
+Action creators for user annotation
+*/
+import _ from "lodash";
+import * as globals from "../globals";
+import { MatrixFBS, AnnotationsHelpers } from "../util/stateManager";
+
+const { isUserAnnotation } = AnnotationsHelpers;
+
+export const annotationCreateCategoryAction = (
+  newCategoryName,
+  categoryToDuplicate
+) => async (dispatch, getState) => {
+  /*
+  Add a new user-created category to the obs annotations.
+
+  Arguments:
+    newCategoryName - string name for the category.
+    categoryToDuplicate - obs category to use for initial values, or null.
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
+  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+  const { schema } = prevAnnoMatrix;
+
+  /* name must be a string,  non-zero length */
+  if (typeof newCategoryName !== "string" || newCategoryName.length === 0)
+    throw new Error("user annotations require string name");
+
+  /* ensure the name isn't already in use! */
+  if (schema.annotations.obsByName[newCategoryName])
+    throw new Error("name collision on annotation category create");
+
+  let initialValue;
+  let newSchema;
+  let ctor;
+  if (categoryToDuplicate) {
+    /* if we are duplicating a category, retrieve it */
+    const catDupSchema = schema.annotations.obsByName[categoryToDuplicate];
+    const catDupType = catDupSchema?.type;
+    if (catDupType !== "string" && catDupType !== "categorical")
+      throw new Error("categoryToDuplicate does not exist or has invalid type");
+
+    const catToDupDf = await prevAnnoMatrix
+      .base()
+      .fetch("obs", categoryToDuplicate);
+    const col = catToDupDf.col(categoryToDuplicate);
+    initialValue = col.asArray();
+    const { categories } = col.summarizeCategorical();
+    // all user-created annotations must have the unassigned category
+    if (!categories.includes(globals.unassignedCategoryLabel)) {
+      categories.push(globals.unassignedCategoryLabel);
+    }
+    ctor = initialValue.constructor;
+    newSchema = {
+      ...catDupSchema,
+      name: newCategoryName,
+      categories,
+      writable: true,
+    };
+  } else {
+    /* else assign to the standard default value */
+    initialValue = globals.unassignedCategoryLabel;
+    ctor = Array;
+    newSchema = {
+      name: newCategoryName,
+      type: "categorical",
+      categories: [globals.unassignedCategoryLabel],
+      writable: true,
+    };
+  }
+
+  const obsCrossfilter = prevObsCrossfilter.addObsColumn(
+    newSchema,
+    ctor,
+    initialValue
+  );
+
+  dispatch({
+    type: "annotation: create category",
+    data: newCategoryName,
+    categoryToDuplicate,
+    annoMatrix: obsCrossfilter.annoMatrix,
+    obsCrossfilter,
+  });
+};
+
+export const annotationRenameCategoryAction = (
+  oldCategoryName,
+  newCategoryName
+) => (dispatch, getState) => {
+  /*
+  Rename a user-created annotation category
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
+  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+  if (!isUserAnnotation(prevAnnoMatrix, oldCategoryName))
+    throw new Error("not a user annotation");
+
+  /* name must be a string,  non-zero length */
+  if (typeof newCategoryName !== "string" || newCategoryName.length === 0)
+    throw new Error("user annotations require string name");
+
+  if (oldCategoryName === newCategoryName) return;
+
+  const obsCrossfilter = prevObsCrossfilter.renameObsColumn(
+    oldCategoryName,
+    newCategoryName
+  );
+
+  dispatch({
+    type: "annotation: category edited",
+    annoMatrix: obsCrossfilter.annoMatrix,
+    obsCrossfilter,
+    metadataField: oldCategoryName,
+    newCategoryText: newCategoryName,
+    data: newCategoryName,
+  });
+};
+
+export const annotationDeleteCategoryAction = (categoryName) => (
+  dispatch,
+  getState
+) => {
+  /*
+  Delete a user-created category
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
+  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+    throw new Error("not a user annotation");
+
+  const obsCrossfilter = prevObsCrossfilter.dropObsColumn(categoryName);
+  dispatch({
+    type: "annotation: delete category",
+    annoMatrix: obsCrossfilter.annoMatrix,
+    obsCrossfilter,
+    metadataField: categoryName,
+  });
+};
+
+export const annotationCreateLabelInCategory = (
+  categoryName,
+  labelName,
+  assignSelected
+) => async (dispatch, getState) => {
+  /*
+  Add a new label to a user-defined category.  If assignSelected is true, assign 
+  the label to all currently selected cells.
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
+  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+    throw new Error("not a user annotation");
+
+  let obsCrossfilter = prevObsCrossfilter.addObsAnnoCategory(
+    categoryName,
+    labelName
+  );
+  if (assignSelected) {
+    obsCrossfilter = await obsCrossfilter.setObsColumnValues(
+      categoryName,
+      prevObsCrossfilter.allSelectedLabels(),
+      labelName
+    );
+  }
+
+  dispatch({
+    type: "annotation: add new label to category",
+    annoMatrix: obsCrossfilter.annoMatrix,
+    obsCrossfilter,
+    metadataField: categoryName,
+    newLabelText: labelName,
+    assignSelectedCells: assignSelected,
+  });
+};
+
+export const annotationDeleteLabelFromCategory = (
+  categoryName,
+  labelName
+) => async (dispatch, getState) => {
+  /*
+  delete a label from a user-defined category
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
+  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+    throw new Error("not a user annotation");
+
+  const obsCrossfilter = await prevObsCrossfilter.removeObsAnnoCategory(
+    categoryName,
+    labelName,
+    globals.unassignedCategoryLabel
+  );
+
+  dispatch({
+    type: "annotation: delete label",
+    metadataField: categoryName,
+    label: labelName,
+    annoMatrix: obsCrossfilter.annoMatrix,
+    obsCrossfilter,
+  });
+};
+
+export const annotationRenameLabelInCategory = (
+  categoryName,
+  oldLabelName,
+  newLabelName
+) => async (dispatch, getState) => {
+  /*
+  label name change
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
+  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+    throw new Error("not a user annotation");
+
+  let obsCrossfilter = await prevObsCrossfilter.resetObsColumnValues(
+    categoryName,
+    oldLabelName,
+    newLabelName
+  );
+  obsCrossfilter = await obsCrossfilter.removeObsAnnoCategory(
+    categoryName,
+    oldLabelName,
+    globals.unassignedCategoryLabel
+  );
+
+  dispatch({
+    type: "annotation: label edited",
+    editedLabel: newLabelName,
+    metadataField: categoryName,
+    label: oldLabelName,
+    annoMatrix: obsCrossfilter.annoMatrix,
+    obsCrossfilter,
+  });
+};
+
+export const annotationLabelCurrentSelection = (
+  categoryName,
+  labelName
+) => async (dispatch, getState) => {
+  /*
+  set the label on all currently selected
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
+  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+    throw new Error("not a user annotation");
+
+  const obsCrossfilter = await prevObsCrossfilter.setObsColumnValues(
+    categoryName,
+    prevObsCrossfilter.allSelectedLabels(),
+    labelName
+  );
+
+  dispatch({
+    type: "annotation: label current cell selection",
+    metadataField: categoryName,
+    label: labelName,
+    obsCrossfilter,
+    annoMatrix: obsCrossfilter.annoMatrix,
+  });
+};
+
+function writableAnnotations(annoMatrix) {
+  return annoMatrix.schema.annotations.obs.columns
+    .filter((s) => s.writable)
+    .map((s) => s.name);
+}
+
+export const needToSaveObsAnnotations = (annoMatrix, lastSavedAnnoMatrix) => {
+  /*
+  Return true if there are LIKELY user-defined annotation modifications between the two
+  annoMatrices.  Technically not an action creator, but intimately intertwined
+  with the save process.
+
+  Two conditions will trigger a need to save:
+    * the collection of user-defined columns have changed
+    * the contents of the user-defined columns have change
+  */
+
+  annoMatrix = annoMatrix.base();
+
+  // if the annoMatrix hasn't changed, we are guaranteed no changes to the matrix schema or contents.
+  if (annoMatrix === lastSavedAnnoMatrix) return false;
+
+  // if the schema has changed, we need to save
+  const currentWritable = writableAnnotations(annoMatrix);
+  if (_.difference(currentWritable, writableAnnotations(lastSavedAnnoMatrix))) {
+    return true;
+  }
+
+  // no schema changes; check for change in contents
+  return currentWritable.some(
+    (col) => annoMatrix.col(col) !== lastSavedAnnoMatrix.col(col)
+  );
+};
+
+export const saveObsAnnotationsAction = () => async (dispatch, getState) => {
+  /*
+  Save the user-created obs annotations IF any have changed.
+  */
+  const state = getState();
+  const { annotations, autosave } = state;
+  const { dataCollectionNameIsReadOnly, dataCollectionName } = annotations;
+  const { lastSavedAnnoMatrix, saveInProgress } = autosave;
+
+  const annoMatrix = state.annoMatrix.base();
+
+  if (saveInProgress || annoMatrix === lastSavedAnnoMatrix) return;
+  if (!needToSaveObsAnnotations(annoMatrix, lastSavedAnnoMatrix)) {
+    dispatch({
+      type: "writable obs annotations - save complete",
+      lastSavedAnnoMatrix: annoMatrix,
+    });
+    return;
+  }
+
+  /*
+  Else, we really do need to save
+  */
+
+  dispatch({
+    type: "writable obs annotations - save started",
+  });
+
+  const df = await annoMatrix.fetch("obs", writableAnnotations(annoMatrix));
+  const matrix = MatrixFBS.encodeMatrixFBS(df);
+  try {
+    const queryString =
+      !dataCollectionNameIsReadOnly && !!dataCollectionName
+        ? `?annotation-collection-name=${encodeURIComponent(
+            dataCollectionName
+          )}`
+        : "";
+    const res = await fetch(
+      `${globals.API.prefix}${globals.API.version}annotations/obs${queryString}`,
+      {
+        method: "PUT",
+        body: matrix,
+        headers: new Headers({
+          "Content-Type": "application/octet-stream",
+        }),
+        credentials: "include",
+      }
+    );
+    if (res.ok) {
+      dispatch({
+        type: "writable obs annotations - save complete",
+        lastSavedAnnoMatrix: annoMatrix,
+      });
+    } else {
+      dispatch({
+        type: "writable obs annotations - save error",
+        message: `HTTP error ${res.status} - ${res.statusText}`,
+        res,
+      });
+    }
+  } catch (error) {
+    dispatch({
+      type: "writable obs annotations - save error",
+      message: error.toString(),
+      error,
+    });
+  }
+};

client/src/actions/embedding.js

@@ -0,0 +1,51 @@
+/*
+action creators related to embeddings choice
+*/
+
+import { AnnoMatrixObsCrossfilter } from "../annoMatrix";
+import { _setEmbeddingSubset } from "../util/stateManager/viewStackHelpers";
+
+export async function _switchEmbedding(
+  prevAnnoMatrix,
+  prevCrossfilter,
+  newEmbeddingName
+) {
+  /*
+  DRY helper used by this and reembedding action creators
+  */
+  const base = prevAnnoMatrix.base();
+  const embeddingDf = await base.fetch("emb", newEmbeddingName);
+  const annoMatrix = _setEmbeddingSubset(prevAnnoMatrix, embeddingDf);
+  const obsCrossfilter = await new AnnoMatrixObsCrossfilter(
+    annoMatrix,
+    prevCrossfilter.obsCrossfilter
+  ).select("emb", newEmbeddingName, {
+    mode: "all",
+  });
+  return [annoMatrix, obsCrossfilter];
+}
+
+export const layoutChoiceAction = (newLayoutChoice) => async (
+  dispatch,
+  getState
+) => {
+  /*
+  On layout choice, make sure we have selected all on the previous layout, AND the new
+  layout.
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevCrossfilter,
+  } = getState();
+  const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
+    prevAnnoMatrix,
+    prevCrossfilter,
+    newLayoutChoice
+  );
+  dispatch({
+    type: "set layout choice",
+    layoutChoice: newLayoutChoice,
+    obsCrossfilter,
+    annoMatrix,
+  });
+};

client/src/actions/index.js

@@ -1,216 +1,119 @@
-// jshint esversion: 6
-import _ from "lodash";
 import * as globals from "../globals";
-import { Universe, MatrixFBS } from "../util/stateManager";
+import { AnnoMatrixLoader, AnnoMatrixObsCrossfilter } from "../annoMatrix";
 import {
   catchErrorsWrap,
   doJsonRequest,
-  doBinaryRequest,
-  dispatchNetworkErrorMessageToUser
+  dispatchNetworkErrorMessageToUser,
 } from "../util/actionHelpers";
+import {
+  requestReembed /* , reembedResetWorldToUniverse -- disabled temporarily, TODO issue #1606 */,
+} from "./reembed";
+import { loadUserColorConfig } from "../util/stateManager/colorHelpers";
+import * as selnActions from "./selection";
+import * as annoActions from "./annotation";
+import * as viewActions from "./viewStack";
+import * as embActions from "./embedding";
 
 /*
-Bootstrap application with the initial data loading.
-  * /config - application configuration
-  * /schema - schema of dataframe
-  * /annotations - all metadata annotation
-  * /layout - all default layout
+return promise fetching user-configured colors
+*/
+async function userColorsFetchAndLoad(dispatch) {
+  return fetchJson("colors").then((response) =>
+    dispatch({
+      type: "universe: user color load success",
+      userColors: loadUserColorConfig(response),
+    })
+  );
+}
+
+async function schemaFetch() {
+  return fetchJson("schema");
+}
+
+async function configFetch(dispatch) {
+  return fetchJson("config").then((response) => {
+    const config = { ...globals.configDefaults, ...response.config };
+    dispatch({
+      type: "configuration load complete",
+      config,
+    });
+    return config;
+  });
+}
+
+async function userInfoFetch(dispatch) {
+  return fetchJson("userinfo").then((response) => {
+    const { userinfo: userInfo } = response || {};
+    dispatch({
+      type: "userInfo load complete",
+      userInfo,
+    });
+    return userInfo;
+  });
+}
+
+function prefetchEmbeddings(annoMatrix) {
+  /*
+  prefetch requests for all embeddings
+  */
+  const { schema } = annoMatrix;
+  const available = schema.layout.obs.map((v) => v.name);
+  available.forEach((embName) => annoMatrix.prefetch("emb", embName));
+}
+
+/*
+Application bootstrap
 */
 const doInitialDataLoad = () =>
-  catchErrorsWrap(async dispatch => {
+  catchErrorsWrap(async (dispatch) => {
     dispatch({ type: "initial data load start" });
 
     try {
-      /*
-      Step 1 - config & schema, all JSON
-      */
-      const requestJson = ["config", "schema"]
-        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
-        .map(url => doJsonRequest(url));
-      const stepOneResults = await Promise.all(requestJson);
-      /* set config defaults */
-      const config = { ...globals.configDefaults, ...stepOneResults[0].config };
-      const schema = stepOneResults[1];
+      const [config, schema] = await Promise.all([
+        configFetch(dispatch),
+        schemaFetch(dispatch),
+        userColorsFetchAndLoad(dispatch),
+        userInfoFetch(dispatch),
+      ]);
 
-      /*
-      Step 2 - dataframes, all binary.  NOTE: uses results of step 1.
-      */
-      /* only load names for var annotations, if possible*/
-      const varIndexName = schema?.schema?.annotations?.var?.index;
-      const varAnnotationsQuery = varIndexName
-        ? `?annotation-name=${encodeURIComponent(varIndexName)}`
-        : "";
-      const varAnnotationsURL = `annotations/var${varAnnotationsQuery}`;
-      const requestBinary = ["annotations/obs", varAnnotationsURL, "layout/obs"]
-        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
-        .map(url => doBinaryRequest(url));
-      const stepTwoResults = await Promise.all(requestBinary);
-      const [obsAnno, varAnno, obsLayout] = [...stepTwoResults];
-
-      const universe = Universe.createUniverseFromResponse(
-        config,
-        schema,
-        obsAnno,
-        varAnno,
-        obsLayout
-      );
+      const baseDataUrl = `${globals.API.prefix}${globals.API.version}`;
+      const annoMatrix = new AnnoMatrixLoader(baseDataUrl, schema.schema);
+      const obsCrossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
+      prefetchEmbeddings(annoMatrix);
 
       dispatch({
-        type: "configuration load complete",
-        config
-      });
-      dispatch({
-        type: "initial data load complete (universe exists)",
-        universe
+        type: "annoMatrix: init complete",
+        annoMatrix,
+        obsCrossfilter,
       });
+      dispatch({ type: "initial data load complete" });
+
+      const defaultEmbedding = config?.parameters?.["default_embedding"];
+      const layoutSchema = schema?.schema?.layout?.obs ?? [];
+      if (
+        defaultEmbedding &&
+        layoutSchema.some((s) => s.name === defaultEmbedding)
+      ) {
+        dispatch(embActions.layoutChoiceAction(defaultEmbedding));
+      }
     } catch (error) {
       dispatch({ type: "initial data load error", error });
     }
   }, true);
 
-/*
-Set the view (world) to current selection.   Placeholder for an async action
-which also does re-layout.
-*/
-const regraph = () => (dispatch, getState) => {
-  const { universe, world, crossfilter } = getState();
-  dispatch({
-    type: "set World to current selection",
-    universe,
-    world,
-    crossfilter
-  });
-};
-
-// Throws
-const dispatchExpressionErrors = (dispatch, res) => {
-  const msg = `Unexpected HTTP response while fetching expression data ${res.status}, ${res.statusText}`;
-  dispatchNetworkErrorMessageToUser(msg);
-  throw new Error(msg);
-};
-
-/*
-Fetch expression vectors for each gene in genes.   This is NOT an action
-function, but rather a helper to be called from an action helper that
-needs expression data.
-
-Transparently utilizes cached data if it is already present.
-*/
-async function _doRequestExpressionData(dispatch, getState, genes) {
-  const state = getState();
-  const { universe } = state;
-  const varIndexName = universe.schema.annotations.var.index;
-
-  /* helper for this function only */
-  const fetchData = async geneNames => {
-    const res = await fetch(
-      `${globals.API.prefix}${globals.API.version}data/var`,
-      {
-        method: "PUT",
-        body: JSON.stringify({
-          filter: {
-            var: {
-              annotation_value: [{ name: varIndexName, values: geneNames }]
-            }
-          }
-        }),
-        headers: new Headers({
-          accept: "application/octet-stream",
-          "Content-Type": "application/json"
-        }),
-        credentials: "include"
-      }
-    );
-
-    if (
-      !res.ok ||
-      res.headers.get("Content-Type") !== "application/octet-stream"
-    ) {
-      // WILL throw
-      return dispatchExpressionErrors(dispatch, res);
-    }
-
-    const data = await res.arrayBuffer();
-    return Universe.convertDataFBStoObject(universe, data);
-  };
-
-  /* preload data already in cache */
-  let expressionData = _.transform(
-    genes,
-    (expData, g) => {
-      const data = universe.varData.col(g);
-      if (data) {
-        expData[g] = data.asArray();
-      }
-    },
-    {}
-  ); // --> { gene: data }
-  /* make a list of genes for which we do not have data */
-  const genesToFetch = _.filter(genes, g => expressionData[g] === undefined);
-
-  dispatch({ type: "expression load start" });
-
-  /* Fetch data for any genes not in cache */
-  if (genesToFetch.length) {
-    try {
-      const newExpressionData = await fetchData(genesToFetch);
-      expressionData = {
-        ...expressionData,
-        ...newExpressionData
-      };
-    } catch (error) {
-      dispatch({ type: "expression load error", error });
-      throw error; // rethrow
-    }
-  }
-
-  dispatch({ type: "expression load success", expressionData });
-  return expressionData;
-}
-
 function requestSingleGeneExpressionCountsForColoringPOST(gene) {
-  return async (dispatch, getState) => {
-    dispatch({ type: "get single gene expression for coloring started" });
-    try {
-      await _doRequestExpressionData(dispatch, getState, [gene]);
-      const { world } = getState();
-      dispatch({
-        type: "color by expression",
-        gene,
-        data: {
-          [gene]: world.varData.col(gene).asArray()
-        }
-      });
-    } catch (error) {
-      dispatch({
-        type: "get single gene expression for coloring error",
-        error
-      });
-    }
+  return {
+    type: "color by expression",
+    gene,
   };
 }
 
-const requestUserDefinedGene = gene => async (dispatch, getState) => {
-  dispatch({ type: "request user defined gene started" });
-  try {
-    await await _doRequestExpressionData(dispatch, getState, [gene]);
-    const { world } = getState();
-
-    /* then send the success case action through */
-    return dispatch({
-      type: "request user defined gene success",
-      data: {
-        genes: [gene],
-        expression: world.varData.col(gene).asArray()
-      }
-    });
-  } catch (error) {
-    return dispatch({
-      type: "request user defined gene error",
-      error
-    });
-  }
-};
+const requestUserDefinedGene = (gene) => ({
+  type: "request user defined gene success",
+  data: {
+    genes: [gene],
+  },
+});
 
 const dispatchDiffExpErrors = (dispatch, response) => {
   switch (response.status) {
@@ -229,7 +132,7 @@ const dispatchDiffExpErrors = (dispatch, response) => {
       dispatchNetworkErrorMessageToUser(msg);
       dispatch({
         type: "request differential expression error",
-        error: new Error(msg)
+        error: new Error(msg),
       });
     }
   }
@@ -246,9 +149,8 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
     1. get the most differentially expressed genes
     2. get expression data for each
     */
-    const state = getState();
-    const { universe } = state;
-    const varIndexName = universe.schema.annotations.var.index;
+    const { annoMatrix } = getState();
+    const varIndexName = annoMatrix.schema.annotations.var.index;
 
     // Legal values are null, Array or TypedArray.  Null is initial state.
     if (!set1) set1 = [];
@@ -267,15 +169,15 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
         method: "POST",
         headers: new Headers({
           Accept: "application/json",
-          "Content-Type": "application/json"
+          "Content-Type": "application/json",
         }),
         body: JSON.stringify({
           mode: "topN",
           count: num_genes,
           set1: { filter: { obs: { index: set1 } } },
-          set2: { filter: { obs: { index: set2 } } }
+          set2: { filter: { obs: { index: set2 } } },
         }),
-        credentials: "include"
+        credentials: "include",
       }
     );
 
@@ -283,123 +185,64 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
       return dispatchDiffExpErrors(dispatch, res);
     }
 
-    const data = await res.json();
-    // result is [ [varIdx, ...], ... ]
-    const topNGenes = _.map(data, r =>
-      universe.varAnnotations.at(r[0], varIndexName)
-    );
-
-    /*
-    Kick off secondary action to fetch all of the expression data for the
-    topN expressed genes.
-    */
-    await _doRequestExpressionData(dispatch, getState, topNGenes);
+    const response = await res.json();
+    const varIndex = await annoMatrix.fetch("var", varIndexName);
+    const data = response.map((v) => [
+      varIndex.at(v[0], varIndexName),
+      ...v.slice(1),
+    ]);
 
     /* then send the success case action through */
     return dispatch({
       type: "request differential expression success",
-      data
+      data,
     });
   } catch (error) {
     return dispatch({
       type: "request differential expression error",
-      error
+      error,
     });
   }
 };
 
-const resetInterface = () => (dispatch, getState) => {
-  const { universe } = getState();
-
-  dispatch({
-    type: "user reset start"
-  });
-  dispatch({
-    type: "clear all user defined genes"
-  });
-  dispatch({
-    type: "clear differential expression"
-  });
-  dispatch({
-    type: "reset colorscale"
-  });
-  dispatch({
-    type: "reset centroid labels"
-  });
-  dispatch({
-    type: "clear scatterplot"
-  });
-  dispatch({
-    type: "reset World to eq Universe",
-    universe
-  });
-  dispatch({
-    type: "increment graph render counter"
-  });
-  dispatch({
-    type: "user reset end"
-  });
-};
-
-const saveObsAnnotations = () => async (dispatch, getState) => {
-  const { universe, annotations } = getState();
-  const { obsAnnotations, schema } = universe;
-  const { dataCollectionNameIsReadOnly, dataCollectionName } = annotations;
-
-  dispatch({
-    type: "writable obs annotations - save started"
-  });
-
-  const writableAnnotations = schema.annotations.obs.columns
-    .filter(s => s.writable)
-    .map(s => s.name);
-  const df = obsAnnotations.subset(null, writableAnnotations);
-  const matrix = MatrixFBS.encodeMatrixFBS(df);
-  try {
-    const queryString =
-      !dataCollectionNameIsReadOnly && !!dataCollectionName
-        ? `?annotation-collection-name=${encodeURIComponent(
-            dataCollectionName
-          )}`
-        : "";
-    const res = await fetch(
-      `${globals.API.prefix}${globals.API.version}annotations/obs${queryString}`,
-      {
-        method: "PUT",
-        body: matrix,
-        headers: new Headers({
-          "Content-Type": "application/octet-stream"
-        }),
-        credentials: "include"
-      }
-    );
-    if (res.ok) {
-      dispatch({
-        type: "writable obs annotations - save complete",
-        obsAnnotations
-      });
-    } else {
-      dispatch({
-        type: "writable obs annotations - save error",
-        message: `HTTP error ${res.status} - ${res.statusText}`,
-        res
-      });
-    }
-  } catch (error) {
-    dispatch({
-      type: "writable obs annotations - save error",
-      message: error.toString(),
-      error
-    });
-  }
-};
+function fetchJson(pathAndQuery) {
+  return doJsonRequest(
+    `${globals.API.prefix}${globals.API.version}${pathAndQuery}`
+  );
+}
 
 export default {
-  regraph,
-  resetInterface,
-  requestSingleGeneExpressionCountsForColoringPOST,
-  requestDifferentialExpression,
-  requestUserDefinedGene,
   doInitialDataLoad,
-  saveObsAnnotations
+  requestDifferentialExpression,
+  requestSingleGeneExpressionCountsForColoringPOST,
+  requestUserDefinedGene,
+  requestReembed,
+  selectContinuousMetadataAction: selnActions.selectContinuousMetadataAction,
+  selectCategoricalMetadataAction: selnActions.selectCategoricalMetadataAction,
+  selectCategoricalAllMetadataAction:
+    selnActions.selectCategoricalAllMetadataAction,
+  graphBrushStartAction: selnActions.graphBrushStartAction,
+  graphBrushChangeAction: selnActions.graphBrushChangeAction,
+  graphBrushDeselectAction: selnActions.graphBrushDeselectAction,
+  graphBrushCancelAction: selnActions.graphBrushCancelAction,
+  graphBrushEndAction: selnActions.graphBrushEndAction,
+  graphLassoStartAction: selnActions.graphLassoStartAction,
+  graphLassoEndAction: selnActions.graphLassoEndAction,
+  graphLassoCancelAction: selnActions.graphLassoCancelAction,
+  graphLassoDeselectAction: selnActions.graphLassoDeselectAction,
+  clipAction: viewActions.clipAction,
+  subsetAction: viewActions.subsetAction,
+  resetSubsetAction: viewActions.resetSubsetAction,
+  annotationCreateCategoryAction: annoActions.annotationCreateCategoryAction,
+  annotationRenameCategoryAction: annoActions.annotationRenameCategoryAction,
+  annotationDeleteCategoryAction: annoActions.annotationDeleteCategoryAction,
+  annotationCreateLabelInCategory: annoActions.annotationCreateLabelInCategory,
+  annotationDeleteLabelFromCategory:
+    annoActions.annotationDeleteLabelFromCategory,
+  annotationRenameLabelInCategory: annoActions.annotationRenameLabelInCategory,
+  annotationLabelCurrentSelection: annoActions.annotationLabelCurrentSelection,
+  saveObsAnnotationsAction: annoActions.saveObsAnnotationsAction,
+  needToSaveObsAnnotations: annoActions.needToSaveObsAnnotations,
+  layoutChoiceAction: embActions.layoutChoiceAction,
+  setCellSetFromSelection: selnActions.setCellSetFromSelection,
 };

client/src/actions/reembed.js

@@ -0,0 +1,109 @@
+import { API } from "../globals";
+import {
+  postNetworkErrorToast,
+  postAsyncSuccessToast,
+  postAsyncFailureToast,
+} from "../components/framework/toasters";
+import { _switchEmbedding } from "./embedding";
+
+function abortableFetch(request, opts, timeout = 0) {
+  const controller = new AbortController();
+  const { signal } = controller;
+
+  return {
+    abort: () => controller.abort(),
+    isAborted: () => signal.aborted,
+    ready: () => {
+      if (timeout) {
+        setTimeout(() => controller.abort(), timeout);
+      }
+      return fetch(request, { ...opts, signal });
+    },
+  };
+}
+
+async function doReembedFetch(dispatch, getState) {
+  const state = getState();
+  let cells = state.annoMatrix.rowIndex.labels();
+
+  // These lines ensure that we convert any TypedArray to an Array.
+  // This is necessary because JSON.stringify() does some very strange
+  // things with TypedArrays (they are marshalled to JSON objects, rather
+  // than being marshalled as a JSON array).
+  cells = Array.isArray(cells) ? cells : Array.from(cells);
+
+  const af = abortableFetch(
+    `${API.prefix}${API.version}layout/obs`,
+    {
+      method: "PUT",
+      headers: new Headers({
+        Accept: "application/octet-stream",
+        "Content-Type": "application/json",
+      }),
+      body: JSON.stringify({
+        method: "umap",
+        filter: { obs: { index: cells } },
+      }),
+      credentials: "include",
+    },
+    60000 // 1 minute timeout
+  );
+  dispatch({
+    type: "reembed: request start",
+    abortableFetch: af,
+  });
+  const res = await af.ready();
+
+  if (res.ok && res.headers.get("Content-Type").includes("application/json")) {
+    return res;
+  }
+
+  // else an error
+  let msg = `Unexpected HTTP response ${res.status}, ${res.statusText}`;
+  const body = await res.text();
+  if (body && body.length > 0) {
+    msg = `${msg} -- ${body}`;
+  }
+  throw new Error(msg);
+}
+
+/*
+functions below are dispatch-able
+*/
+export function requestReembed() {
+  return async (dispatch, getState) => {
+    try {
+      const res = await doReembedFetch(dispatch, getState);
+      const schema = await res.json();
+      dispatch({
+        type: "reembed: request completed",
+      });
+
+      const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevCrossfilter } = getState();
+      const base = prevAnnoMatrix.base().addEmbedding(schema);
+      const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
+        base,
+        prevCrossfilter,
+        schema.name
+      );
+      dispatch({
+        type: "reembed: add reembedding",
+        schema,
+        annoMatrix,
+        obsCrossfilter,
+      });
+
+      postAsyncSuccessToast("Re-embedding has completed.");
+    } catch (error) {
+      dispatch({
+        type: "reembed: request aborted",
+      });
+      if (error.name === "AbortError") {
+        postAsyncFailureToast("Re-embedding calculation was aborted.");
+      } else {
+        postNetworkErrorToast(`Re-embedding: ${error.message}`);
+      }
+      console.log("Reembed exception:", error, error.name, error.message);
+    }
+  };
+}

client/src/actions/selection.js

@@ -0,0 +1,199 @@
+/*
+Action creators for selection 
+*/
+export const selectContinuousMetadataAction = (
+  type,
+  query,
+  range,
+  oldProps = {}
+) => async (dispatch, getState) => {
+  const { obsCrossfilter: prevObsCrossfilter } = getState();
+
+  const selection = range
+    ? {
+        mode: "range",
+        lo: range[0],
+        hi: range[1],
+        inclusive: true, // [lo, hi] incluisve selection
+      }
+    : { mode: "all" };
+
+  const obsCrossfilter = await prevObsCrossfilter.select(...query, selection);
+
+  dispatch({
+    type,
+    obsCrossfilter,
+    range,
+    ...oldProps,
+  });
+};
+
+export const selectCategoricalMetadataAction = (
+  type, // action type
+  metadataField, // annotation category name
+  labels,
+  label, // the label being selected/deselected
+  isSelected, // bool
+  oldProps = {}
+) => async (dispatch, getState) => {
+  const {
+    obsCrossfilter: prevObsCrossfilter,
+    categoricalSelection,
+  } = getState();
+
+  const labelSelectionState = new Map(categoricalSelection[metadataField]);
+  labels.forEach(
+    (l) => labelSelectionState.has(l) || labelSelectionState.set(l, true)
+  );
+  labelSelectionState.set(label, isSelected);
+
+  const values = Array.from(labelSelectionState.keys()).filter((k) =>
+    labelSelectionState.get(k)
+  );
+  const selection = {
+    mode: "exact",
+    values,
+  };
+  const obsCrossfilter = await prevObsCrossfilter.select(
+    "obs",
+    metadataField,
+    selection
+  );
+
+  dispatch({
+    type,
+    obsCrossfilter,
+    metadataField,
+    labelSelectionState,
+    ...oldProps,
+  });
+};
+
+export const selectCategoricalAllMetadataAction = (
+  type, // action type
+  metadataField, // annotation category name
+  labels,
+  isSelected, // bool, select all or none
+  oldProps = {}
+) => async (dispatch, getState) => {
+  const {
+    obsCrossfilter: prevObsCrossfilter,
+    categoricalSelection,
+  } = getState();
+
+  const labelSelectionState = new Map(categoricalSelection[metadataField]);
+  labels.forEach((label) => labelSelectionState.set(label, isSelected));
+
+  const selection = { mode: isSelected ? "all" : "none" };
+  const obsCrossfilter = await prevObsCrossfilter.select(
+    "obs",
+    metadataField,
+    selection
+  );
+
+  dispatch({
+    type,
+    obsCrossfilter,
+    metadataField,
+    labelSelectionState,
+    ...oldProps,
+  });
+};
+
+/**
+ ** Graph selection-related actions
+ **/
+
+export const graphBrushStartAction = () =>
+  /* no change to crossfilter until a change fires */
+  ({ type: "graph brush start" });
+
+const _graphBrushWithinRectAction = (type, embName, brushCoords) => async (
+  dispatch,
+  getState
+) => {
+  const { obsCrossfilter: prevObsCrossfilter } = getState();
+
+  const selection = { mode: "within-rect", ...brushCoords };
+  const obsCrossfilter = await prevObsCrossfilter.select(
+    "emb",
+    embName,
+    selection
+  );
+
+  dispatch({
+    type,
+    obsCrossfilter,
+    brushCoords,
+  });
+};
+
+const _graphAllAction = (type, embName) => async (dispatch, getState) => {
+  const { obsCrossfilter: prevObsCrossfilter } = getState();
+
+  const obsCrossfilter = await prevObsCrossfilter.select("emb", embName, {
+    mode: "all",
+  });
+
+  dispatch({
+    type,
+    obsCrossfilter,
+  });
+};
+
+export const graphBrushChangeAction = (embName, brushCoords) =>
+  _graphBrushWithinRectAction("graph brush change", embName, brushCoords);
+
+export const graphBrushEndAction = (embName, brushCoords) =>
+  _graphBrushWithinRectAction("graph brush end", embName, brushCoords);
+
+export const graphBrushCancelAction = (embName) =>
+  _graphAllAction("graph brush cancel", embName);
+export const graphBrushDeselectAction = (embName) =>
+  _graphAllAction("graph brush deselect", embName);
+
+export const graphLassoStartAction = () =>
+  /* no change to crossfilter until a change fires */
+  ({ type: "graph lasso start" });
+
+export const graphLassoCancelAction = (embName) =>
+  _graphAllAction("graph lasso cancel", embName);
+
+export const graphLassoDeselectAction = (embName) =>
+  _graphAllAction("graph lasso cancel", embName);
+
+export const graphLassoEndAction = (embName, polygon) => async (
+  dispatch,
+  getState
+) => {
+  const { obsCrossfilter: prevObsCrossfilter } = getState();
+
+  const selection = {
+    mode: "within-polygon",
+    polygon,
+  };
+  const obsCrossfilter = await prevObsCrossfilter.select(
+    "emb",
+    embName,
+    selection
+  );
+
+  dispatch({
+    type: "graph lasso end",
+    obsCrossfilter,
+    polygon,
+  });
+};
+
+/*
+Differential expression set selection
+*/
+export const setCellSetFromSelection = (cellSetId) => (dispatch, getState) => {
+  const { obsCrossfilter } = getState();
+  const selected = obsCrossfilter.allSelectedLabels();
+
+  dispatch({
+    type: `store current cell selection as differential set ${cellSetId}`,
+    data: selected.length > 0 ? selected : null,
+  });
+};

client/src/actions/viewStack.js

@@ -0,0 +1,76 @@
+/*
+The following actions manage the view stack for annoMatrix.
+
+Conventions used and assumed elsewhere in the code base:
+  * there will be zero or one clip view, and it will be the TOP view always.
+  * there will be zero or more subset views
+
+In other words, in our current use, we do not stack multiple clip views but we do
+stack multiple subsets.
+
+If these conventions change, code elsewhere (eg. menubar/clip.js) will need to
+change as well.
+*/
+import { AnnoMatrixObsCrossfilter } from "../annoMatrix";
+import {
+  _clipAnnoMatrix,
+  _userSubsetAnnoMatrix,
+  _userResetSubsetAnnoMatrix,
+} from "../util/stateManager/viewStackHelpers";
+
+export const clipAction = (min, max) => (dispatch, getState) => {
+  /*
+  apply a clip to the current annoMatrix.  By convention, the clip
+  view is ALWAYS the top view.
+  */
+  const { annoMatrix: prevAnnoMatrix } = getState();
+  const annoMatrix = _clipAnnoMatrix(prevAnnoMatrix, min, max);
+  const obsCrossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
+  dispatch({
+    type: "set clip quantiles",
+    clipQuantiles: { min, max },
+    annoMatrix,
+    obsCrossfilter,
+  });
+};
+
+export const subsetAction = () => (dispatch, getState) => {
+  /*
+  Subset the annoMatrix to the current crossfilter selection by pushing a 
+  subset view.
+
+  By convention, a clip view is ALWAYS the top view, so if present, pop
+  off and re-apply
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
+  const annoMatrix = _userSubsetAnnoMatrix(
+    prevAnnoMatrix,
+    prevObsCrossfilter.allSelectedMask()
+  );
+  const obsCrossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
+  dispatch({
+    type: "subset to selection",
+    annoMatrix,
+    obsCrossfilter,
+  });
+};
+
+export const resetSubsetAction = () => (dispatch, getState) => {
+  /*
+  Reset the annoMatrix to all data.  Because we may have multiple views
+  stacked, we pop them all.  By convention, any clip transformation will
+  be the top of the stack, and must be preserved.
+  */
+
+  const { annoMatrix: prevAnnoMatrix } = getState();
+  const annoMatrix = _userResetSubsetAnnoMatrix(prevAnnoMatrix);
+  const obsCrossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
+  dispatch({
+    type: "reset subset",
+    annoMatrix,
+    obsCrossfilter,
+  });
+};

client/src/annoMatrix/annoMatrix.js

@@ -0,0 +1,658 @@
+import {
+  Dataframe,
+  IdentityInt32Index,
+  dataframeMemo,
+} from "../util/dataframe";
+import {
+  _getColumnDimensionNames,
+  _getColumnSchema,
+  _schemaColumns,
+  _getWritableColumns,
+} from "./schema";
+import { indexEntireSchema } from "../util/stateManager/schemaHelpers";
+import { _whereCacheGet, _whereCacheMerge } from "./whereCache";
+import _shallowClone from "./clone";
+
+const _dataframeCache = dataframeMemo(128);
+
+export default class AnnoMatrix {
+  /*
+  Abstract base class for all AnnoMatrix objects.  This class provides a proxy
+  to the annotated matrix data authoritatively served by the server/back-end.
+
+  AnnoMatrix instances are immutable, meaning that their schema and dimensionality
+  will not change, and simple object equality can be used to detect structural
+  changes. The actual data is cached, and not guaranteed to be present -- any
+  request to access data must be resolved by a fetch() call, which is async, and
+  may involve a server round-trip.
+
+  Guarantees made by the immutabilty, ie, any of these can be detected by
+  simple annoMatrix compare:
+    * schema is the same, including all fields and columns
+    * dimensionality is the same (nObs, nVar)
+    * data mapping/transformation, such as clipping, are the same
+
+  AnnoMatrixes also "stack" like filters, allowing for the construction of
+  views which transform the data in some manner.
+
+  The bootstrap class is AnnoMatrixLoader, which is the caching server proxy, and
+  is bootstrapped with a API URL:
+    new AnnoMatirx(url, schema) -> annoMatrix
+
+  There are various "views", such as AnnoMatrixRowSubsetView, which provide
+  the same interface but with a transformed view of the server data.  Utilities in
+  viewCreators.js can be used to create these views:
+      clip(annoMatrix, min, max) -> annoMatrix
+      subset(annoMatrix, rowLabels) -> annoMatrix
+  etc.
+  */
+  static fields() {
+    /*
+    return the fields present in the AnnoMatrix instance.
+    */
+    return ["obs", "var", "emb", "X"];
+  }
+
+  constructor(schema, nObs, nVar, rowIndex = null) {
+    /*
+    Private constructor - this is an abstract base class.  Do not use.
+    */
+
+    /*
+    Public instance fields:
+      * schema - the matrix schema.  IMPORTANT: always the entire schema, for the
+        base (unfiltered, unclipped, unsubset) annotated matrix, as the server
+        presents it.
+      * nObs, nVar - size of each dimension.  These will accurately reflect the
+        size of the current annoMatrix view.  For example, if you subset the view,
+        the nObs will be smaller.
+      * rowIndex - a rowIndex shared by all data on this view (ie, the list of cells).
+        The row index labels are as defined by the base dataset from the server.
+      * isView - true if this is a view, false if not.
+      * viewOf - pointer to parent annomatrix if a view, undefined/null if not a view.
+      * userFlags - container for any additional state a user of this API wants to hang
+        off of an annoMatrix, and have propagated by the (shallow) cloning protocol.
+    */
+    this.schema = indexEntireSchema(schema);
+    this.nObs = nObs;
+    this.nVar = nVar;
+    this.rowIndex = rowIndex || new IdentityInt32Index(nObs);
+    this.isView = false;
+    this.viewOf = undefined;
+    this.userFlags = {};
+
+    /*
+		Private instance variables.
+
+		These are caches - lazily loaded. The only guarantee is that if they
+		are loaded, they will conform to the schema & dimensionality constraints.
+
+		Do NOT use directly - instead, use the fetch() and preload() API.
+		*/
+    this._cache = {
+      obs: Dataframe.empty(this.rowIndex),
+      var: Dataframe.empty(this.rowIndex),
+      emb: Dataframe.empty(this.rowIndex),
+      X: Dataframe.empty(this.rowIndex),
+    };
+    this._pendingLoad = {
+      obs: {},
+      var: {},
+      emb: {},
+      X: {},
+    };
+    this._whereCache = {};
+    this._gcInfo = new Map();
+  }
+
+  /**
+   ** Schema helper/accessors
+   **/
+  getMatrixColumns(field) {
+    /*
+    Return array of column names in the field.  ONLY supported on the
+    obs, var and emb fields.  X currently unimplemented and will throw.
+
+    For exmaple:
+
+      annoMatrix.getMatrixColumns("obs") -> ["louvain", "n_genes"]
+    */
+    return _schemaColumns(this.schema, field);
+  }
+
+  // eslint-disable-next-line class-methods-use-this -- need to be able to call this on instances
+  getMatrixFields() {
+    /*
+    Return array of fields in this annoMatrix.  Currently hard-wired to
+    return:  ["X", "obs", "var", "emb"].
+
+    These are the fields from data may be requested.
+    */
+    return AnnoMatrix.fields();
+  }
+
+  getColumnSchema(field, col) {
+    /*
+    Return the schema for the field & column ,eg,
+
+      anonMatrix.getColumnSchema("obs", "n_genes") -> { type: "int32", name: "n_genes" }
+
+    This is identical to the information in the annoMatrix.schema
+    instance variable.
+    */
+    return _getColumnSchema(this.schema, field, col);
+  }
+
+  getColumnDimensions(field, col) {
+    /*
+    Return the dimensions on this field / column.  For most fields, which are 1D,
+    this just return the column name.  Multi-dimensional columns, such as embeddings,
+    will return >1 name.
+
+    Examples:
+
+      getColumnDimensions("obs", "louvain") -> ["louvain"]
+      getColumnDimensions("emb", "umap") -> ["umap_0", "umap_1"]
+
+    */
+    return _getColumnDimensionNames(this.schema, field, col);
+  }
+
+  /**
+   ** General utility methods
+   **/
+  base() {
+    /*
+    return the base of view, or `this` if not a view.
+    */
+    let annoMatrix = this;
+    while (annoMatrix.isView) annoMatrix = annoMatrix.viewOf;
+    return annoMatrix;
+  }
+
+  /**
+   ** Load / read interfaces
+   **/
+  fetch(field, q) {
+    /*
+		Return the given query on a single matrix field as a single dataframe.
+		Currently supports ONLY full column query.
+
+		Returns a Promise for the query result, which will resolve to a dataframe.
+
+		Field must be one of the matrix fields: 'obs', 'var', 'X', 'emb'.  Value
+		represents the underlying object upon which the query is occuring.
+
+		Query is one of:
+			* a string, representing a single column name from the field, eg,
+				"n_genes"
+			* an object, containing an "value" query (see below).
+			* an array, containing one or more of the above.
+
+		Columns may have more than one dimension, and all will be fetched
+		and returned together.  This is most commonly seen in an embedding,
+    which usually has two dimensions.
+
+		A value query allows for fetching based upon the value in another 
+		field/column, similar to a join.  Currently only supported on the var
+		dimension, allowing query of X columns by var value (eg, gene name)
+
+    The query filter	is a single value filter:
+			{ "field name": [
+				{name: "column name", values: [ list of values ]}
+			]}
+		One and only one value filter is allowed in a value query.
+
+		Examples:
+
+    1. Fetch the "n_genes" column the "obs":
+
+			const df = await fetch("obs", "n_genes")
+      console.log("Largest number of genes is: ", df.summarize().max);
+
+    2. Fetch two separate columns from obs.  Returns a single dataframe containing
+       the columns:
+
+			const df = await fetch("obs", ["n_genes", "louvain"])
+      console.log("Cell 0 has category: ", df.at(0, "louvain"));
+
+    3. Fetch an entire X (expression counts) column that has a var annotation
+       value "TYMP" in the var index.
+
+			fetch("X", { 
+				where: {field: "var", column: this.schema.annotations.var.index, value: "TYMP"}
+			})
+
+      In AnnData & Pandas DataFrame API, this is equivalent to:
+        adata.X[:, adata.var.index.get_loc("SUMO3")]
+
+		The value query is a recodification and subset of the server REST API 
+		value filter JSON.  Range queries and multiple filters are not currently
+		supported.
+
+		*/
+    return this._fetch(field, q);
+  }
+
+  prefetch(field, q) {
+    /*
+		Start a data fetch & cache fill.  Identical to fetch() except it does
+		not return a value.
+
+    Primary use is to being a cache load as early as is possible, reducing
+    overall component rendering latency.
+		*/
+    this._fetch(field, q);
+    return undefined;
+  }
+
+  /**
+   ** Save / mutate interfaces - manipulation of "writable" OBS annotations.
+   **
+   ** These are all present to support client-side creation of OBS annotations, aka
+   ** "user annotations".
+   **
+   ** They implement common manipulations to the AnnoMatrix, maintaining the
+   ** norma guarantees around correctness of public API, eg,
+   **   - schema will be correct, including the "writable" attribute
+   **   - fetch() will return the latest data, even from views
+   **   - immutability guranteeds
+   **
+   ** As most of these interfaces mutate the annoMatrix, they return a new
+   ** annoMatrix
+   **
+   ** The actual implementation is in the sub-classes, which MUST override these.
+   **/
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  addObsAnnoCategory(col, category) {
+    /*
+    Add a new category value (aka "label") to a writable obs column, and return the new AnnoMatrix.
+    Typical use is to add a new user-created label to a user-created obs categorical
+    annotation.
+
+    Will throw column does not exist or is not writable.
+
+    Example:
+
+      addObsAnnoCategory("my cell type", "left toenail") -> AnnoMatrix
+
+    */
+    _subclassResponsibility();
+  }
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  async removeObsAnnoCategory(col, category, unassignedCategory) {
+    /*
+    Remove a category value from an obs column, reassign any obs having that value
+    to the 'unassignedCategory' value, and return a promise for a new AnnoMatrix.
+    Typical use is to remove a user-created label from a user-created obs categorical
+    annotation.
+
+    Will throw column does not exist or is not writable.
+
+    An `unassignedCategory` value must be provided, for assignment to any obs/cells
+    that had the now-delete category label as their value.
+
+    Example:
+      await removeObsAnnoCategory("my-tissue-type", "right earlobe", "unassigned") -> AnnoMatrix
+
+    NOTE: method is async as it may need to fetch data to provide the reassignment.
+    */
+    _subclassResponsibility();
+  }
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  dropObsColumn(col) {
+    /*
+    Drop an entire writable column, eg a user-created obs annotation.  Typical use
+    is to provide the "Delete Category" implementation.  Returns the new AnnoMatrix.
+    Will throw if not a writable annotation.
+
+    Will throw column does not exist or is not writable.
+
+    Example:
+
+      dropObsColumn("old annotations") ->  AnnoMatrix
+    */
+    _subclassResponsibility();
+  }
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  addObsColumn(colSchema, Ctor, value) {
+    /*
+    Add a new writable OBS annotation column, with the caller-specified schema, initial value
+    type and value.
+
+    Value may be any one of:
+      * an array of values
+      * a primitive type, including null or undefined.
+    If an array, length must be the same as 'this.nObs', and constructor must equal 'Ctor'.
+    If a primitive, 'Ctor' will be used to create the initial value, which will be filled
+    with 'value'.
+
+    Throws if the name specified in 'colSchema' duplicates an existing obs column.
+
+    Returns a new AnnoMatrix.
+
+    Examples:
+
+      addObsColumn(
+        { name: "foo", type: "categorical", categories: "unassigned" },
+        Array,
+        "unassigned"
+      ) -> AnnoMatrix
+
+    */
+    _subclassResponsibility();
+  }
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  renameObsColumn(oldCol, newCol) {
+    /*
+    Rename the obs column 'oldCol' to have name 'newCol' and returns new AnnoMatrix.
+
+    Will throw column does not exist or is not writable, or if 'newCol' is not unique.
+
+    Example:
+
+      renameObsColumn('cell type', 'old cell type') -> AnnoMatrix.
+
+    */
+    _subclassResponsibility();
+  }
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  async setObsColumnValues(col, obsLabels, value) {
+    /*
+    Set all obs with label in array 'obsLabels' to have 'value'.  Typical use would be
+    to set a group of cells to have a label on a user-created categorical anntoation
+    (eg set all selected cells to have a label).
+
+    NOTE: async method, as it may need to fetch.
+
+    Will throw column does not exist or is not writable.
+
+    Example:
+      await setObsColmnValues("flavor", [383, 400], "tasty") -> AnnoMtarix
+
+    */
+    _subclassResponsibility();
+  }
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  async resetObsColumnValues(col, oldValue, newValue) {
+    /*
+    Set by value - all elements in the column with value 'oldValue' are set to 'newValue'.
+    Async method - returns a promise for a new AnnoMatrix.
+
+    Typical use would be to set all labels of one value to another.
+
+    Will throw column does not exist or is not writable.
+
+    Example:
+      await resetObsColumnValues("my notes", "good", "not-good") -> AnnoMatrix
+
+    */
+    _subclassResponsibility();
+  }
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  addEmbedding(colSchema) {
+    /*
+    Add a new obs embedding to the AnnoMatrix, with provided schema.
+    Returns a new annomatrix.
+
+    Typical use will be to add a re-embedding that the server has calculated.
+
+    Will throw if the column schema is invalid (eg, duplicate name).
+    */
+    _subclassResponsibility();
+  }
+
+  /**
+   ** Private interfaces below.
+   **/
+  _resolveCachedQueries(field, queries) {
+    return queries
+      .map((query) =>
+        _whereCacheGet(this._whereCache, this.schema, field, query).filter(
+          (cacheKey) =>
+            cacheKey !== undefined && this._cache[field].hasCol(cacheKey)
+        )
+      )
+      .flat();
+  }
+
+  async _fetch(field, q) {
+    if (!AnnoMatrix.fields().includes(field)) return undefined;
+    const queries = Array.isArray(q) ? q : [q];
+
+    /* find cached columns we need, and GC the rest */
+    const cachedColumns = this._resolveCachedQueries(field, queries);
+    this._gcFetchCleanup(field, cachedColumns);
+
+    /* find any query not already cached */
+    const uncachedQueries = queries.filter((query) =>
+      _whereCacheGet(this._whereCache, this.schema, field, query).some(
+        (cacheKey) =>
+          cacheKey === undefined || !this._cache[field].hasCol(cacheKey)
+      )
+    );
+
+    /* load uncached queries */
+    if (uncachedQueries.length > 0) {
+      await Promise.all(
+        uncachedQueries.map((query) =>
+          this._getPendingLoad(field, query, async (_field, _query) => {
+            /* fetch, then index.  _doLoad is subclass interface */
+            const [whereCacheUpdate, df] = await this._doLoad(_field, _query);
+            this._cache[_field] = this._cache[_field].withColsFrom(df);
+            this._whereCache = _whereCacheMerge(
+              this._whereCache,
+              whereCacheUpdate
+            );
+          })
+        )
+      );
+    }
+
+    /* everything we need is in the cache, so just cherry-pick requested columns */
+    const requestedCacheKeys = this._resolveCachedQueries(field, queries);
+    const response = _dataframeCache(
+      this._cache[field].subset(null, requestedCacheKeys)
+    );
+    this._gcUpdateStats(field, response);
+    return response;
+  }
+
+  async _getPendingLoad(field, query, fetchFn) {
+    /*
+    Given a query on a field, ensure that we only have a single outstanding
+    fetch at any given time.  If multiple requests occur while a fetch is
+    outstanding, just wait for the original.
+
+    This is implemented by returning a promise that will await the singular
+    fetch promise.
+    */
+    const key = _queryCacheKey(field, query);
+    if (!this._pendingLoad[field][key]) {
+      this._pendingLoad[field][key] = fetchFn(field, query);
+      try {
+        await this._pendingLoad[field][key];
+      } finally {
+        delete this._pendingLoad[field][key];
+      }
+    }
+    return this._pendingLoad[field][key];
+  }
+
+  // eslint-disable-next-line class-methods-use-this -- make sure subclass implements
+  async _doLoad() {
+    _subclassResponsibility();
+  }
+
+  /**
+   ** Garbage collection of annomatrix cache to manage memory use.
+   **/
+
+  /*
+  These callbacks implement a GC policy for the cache.  Background:
+
+    * For the Loader (base) annomatrix, re-filling the cache is expensive as
+      it requires an HTTP fetch.
+    * user-defined / writable columns must not be GC'ed as they may be
+      still pending a save/commit.
+    * For views, cost is less and (roughly) proportional with nObs
+    * obs, var and emb do not grow without bounds, and are needed constantly 
+      for rendering.  
+      a) There is no upside to GC'ing these in the base (loader)
+      b) The undo/redo cache can hold a large number in views, which is worht GC'ing
+    * X is often much larger than memory, and the UI allows add/del from
+      this.  Most of the GC potential is here in both the base and views.
+
+  Current policy:
+    * if in active use ("hot") do not GC obs, var or emb.
+    * never, ever GC writable obs columns
+    * For base/loader set a numeric limit on maximum X column count
+    * For views, apply a fixed limit to the number of columns cached in any field.
+      Limit will be lower if not hot.
+
+  To be effective, the GC callback needs to be invoked from the undo/redo code,
+  as much of the cache is pinned by that data structure.
+  */
+  _gcField(field, isHot, pinnedColumns) {
+    const maxColumns = isHot ? 256 : 10; // maybe to aggessive?
+
+    const cache = this._cache[field];
+    if (cache.colIndex.size() < maxColumns) return; // trivial rejection
+
+    const candidates = cache.colIndex
+      .labels()
+      .filter((col) => !pinnedColumns.includes(col));
+
+    const excessCount = candidates.length + pinnedColumns.length - maxColumns;
+    if (excessCount > 0) {
+      const { _gcInfo } = this;
+      candidates.sort((a, b) => {
+        let atime = _gcInfo.get(_columnCacheKey(field, a));
+        if (atime === undefined) atime = 0;
+
+        let btime = _gcInfo.get(_columnCacheKey(field, b));
+        if (btime === undefined) btime = 0;
+
+        return atime - btime;
+      });
+
+      const toDrop = candidates.slice(0, excessCount);
+      // helpful debugging - please leave in place.
+      // console.log(
+      //   `GC: dropping from ${field} hot:${isHot}, columns [${toDrop.join(
+      //     ", "
+      //   )}]`
+      // );
+      this._cache[field] = toDrop.reduce(
+        (df, col) => df.dropCol(col),
+        this._cache[field]
+      );
+      toDrop.forEach((col) => _gcInfo.delete(_columnCacheKey(field, col)));
+    }
+  }
+
+  _gcFetchCleanup(field, pinnedColumns) {
+    /*
+    Called during data load/fetch.  By definition, this is 'hot', so we
+    only want to gc X.
+    */
+    if (field === "X") {
+      this._gcField(
+        field,
+        true,
+        pinnedColumns.concat(_getWritableColumns(this.schema, field))
+      );
+    }
+  }
+
+  _gc(hints) {
+    /*
+    Called from middleware, or elsewhere.  isHot is true if we are in the active store, 
+    or false if we are in some other context (eg, history state).
+    */
+    const { isHot } = hints;
+    const candidateFields = isHot ? ["X"] : ["X", "emb", "var", "obs"];
+    candidateFields.forEach((field) =>
+      this._gcField(field, isHot, _getWritableColumns(this.schema, field))
+    );
+  }
+
+  _gcUpdateStats(field, dataframe) {
+    /*
+    called each time a query is performed, allowing the gc to update any bookkeeping
+    information.  Currently, this is just a simple last-fetched timestamp, stored
+    in a Map.
+
+    Map objects preserve order of insertion. This is leveraged as a cheap way to
+    do LRU, by removing and re-inserting keys.  IMPORTANT: the cleanup code assumes
+    the map insertion order is least-recently-used first.
+    */
+    const cols = dataframe.colIndex.labels();
+    const { _gcInfo } = this;
+    const now = Date.now();
+    cols.forEach((c) => {
+      // gcInfo.delete(c);
+      _gcInfo.set(_columnCacheKey(field, c), now);
+    });
+  }
+
+  /**
+  Cloning sublcass protocol - we rely in cloning to preserve immutable
+  symantics while not causing races or other side effects in internal
+  cache management.
+
+  Subclasses must override _cloneDeeper() if they have state which requires 
+  something other than a shallow copy.  Overrides MUST call super()._cloneDeepr(), 
+  and return its result (after any required modification).  _cloneDeeper()
+  will be called on the OLD object, with the NEW object as an argument.
+
+  Do not override _clone();
+  **/
+  _cloneDeeper(clone) {
+    clone._cache = _shallowClone(this._cache);
+    clone._gcInfo = new Map();
+    clone._pendingLoad = {
+      obs: {},
+      var: {},
+      emb: {},
+      X: {},
+    };
+    return clone;
+  }
+
+  _clone() {
+    const clone = _shallowClone(this);
+    this._cloneDeeper(clone);
+    Object.seal(clone);
+    return clone;
+  }
+}
+
+/*
+private utility functions below
+*/
+
+function _queryCacheKey(field, query) {
+  if (typeof query === "object") {
+    const { field: queryField, column: queryColumn, value: queryValue } = query;
+    return `${field}/${queryField}/${queryColumn}/${queryValue}`;
+  }
+  return `${field}/${query}`;
+}
+
+function _columnCacheKey(field, column) {
+  return `${field}/${column}`;
+}
+
+function _subclassResponsibility() {
+  /* protect against bugs in subclass */
+  throw new Error("subclass failed to implement required method");
+}

client/src/annoMatrix/clone.js

@@ -0,0 +1,6 @@
+/*
+Shallow clone an object, correctly handling prototype
+*/
+export default function _shallowClone(orig) {
+  return Object.assign(Object.create(Object.getPrototypeOf(orig)), orig);
+}

client/src/annoMatrix/crossfilter.js

@@ -0,0 +1,268 @@
+/*
+Row crossfilter proxy for an AnnoMatrix.  This wraps Crossfilter,
+providing a number of services, and ensuring that the crossfilter and
+AnnoMatrix stay in sync:
+	- on-demand index creation as data is loaded
+	- transparently mapping between queries and crossfilter index names.
+  - for mutation of the matrix by user annotations, maintain synchronization
+    between Crossfilter and AnnoMatrix.
+*/
+import Crossfilter from "../util/typedCrossfilter";
+import { _getColumnSchema } from "./schema";
+
+function _dimensionNameFromDf(field, df) {
+  const colNames = df.colIndex.labels();
+  return _dimensionName(field, colNames);
+}
+
+function _dimensionName(field, colNames) {
+  if (!Array.isArray(colNames)) return `${field}/${colNames}`;
+  return `${field}/${colNames.join(":")}`;
+}
+
+export default class AnnoMatrixObsCrossfilter {
+  constructor(annoMatrix, _obsCrossfilter = null) {
+    this.annoMatrix = annoMatrix;
+    this.obsCrossfilter =
+      _obsCrossfilter || new Crossfilter(annoMatrix._cache.obs);
+    this.obsCrossfilter = this.obsCrossfilter.setData(annoMatrix._cache.obs);
+  }
+
+  size() {
+    return this.obsCrossfilter.size();
+  }
+
+  /**
+  Managing the associated annoMatrix.  These wrappers are necessary to 
+  make coordinated changes to BOTH the crossfilter and annoMatrix, and
+  ensure that all state stays synchronized.
+
+  See API documentation in annoMatrix.js.
+  **/
+  addObsColumn(colSchema, Ctor, value) {
+    const annoMatrix = this.annoMatrix.addObsColumn(colSchema, Ctor, value);
+    const obsCrossfilter = this.obsCrossfilter.setData(annoMatrix._cache.obs);
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  dropObsColumn(col) {
+    const annoMatrix = this.annoMatrix.dropObsColumn(col);
+    let { obsCrossfilter } = this;
+    const dimName = _dimensionName("obs", col);
+    if (obsCrossfilter.hasDimension(dimName)) {
+      obsCrossfilter = obsCrossfilter.delDimension(dimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  renameObsColumn(oldCol, newCol) {
+    const annoMatrix = this.annoMatrix.renameObsColumn(oldCol, newCol);
+    const oldDimName = _dimensionName("obs", oldCol);
+    const newDimName = _dimensionName("obs", newCol);
+    let { obsCrossfilter } = this;
+    if (obsCrossfilter.hasDimension(oldDimName)) {
+      obsCrossfilter = obsCrossfilter.renameDimension(oldDimName, newDimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  addObsAnnoCategory(col, category) {
+    const annoMatrix = this.annoMatrix.addObsAnnoCategory(col, category);
+    const dimName = _dimensionName("obs", col);
+    let { obsCrossfilter } = this;
+    if (obsCrossfilter.hasDimension(dimName)) {
+      obsCrossfilter = obsCrossfilter.delDimension(dimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  async removeObsAnnoCategory(col, category, unassignedCategory) {
+    const annoMatrix = await this.annoMatrix.removeObsAnnoCategory(
+      col,
+      category,
+      unassignedCategory
+    );
+    const dimName = _dimensionName("obs", col);
+    let { obsCrossfilter } = this;
+    if (obsCrossfilter.hasDimension(dimName)) {
+      obsCrossfilter = obsCrossfilter.delDimension(dimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  async setObsColumnValues(col, rowLabels, value) {
+    const annoMatrix = await this.annoMatrix.setObsColumnValues(
+      col,
+      rowLabels,
+      value
+    );
+    const dimName = _dimensionName("obs", col);
+    let { obsCrossfilter } = this;
+    if (obsCrossfilter.hasDimension(dimName)) {
+      obsCrossfilter = obsCrossfilter.delDimension(dimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  async resetObsColumnValues(col, oldValue, newValue) {
+    const annoMatrix = await this.annoMatrix.resetObsColumnValues(
+      col,
+      oldValue,
+      newValue
+    );
+    const dimName = _dimensionName("obs", col);
+    let { obsCrossfilter } = this;
+    if (obsCrossfilter.hasDimension(dimName)) {
+      obsCrossfilter = obsCrossfilter.delDimension(dimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  addEmbedding(colSchema) {
+    const annoMatrix = this.annoMatrix.addEmbedding(colSchema);
+    return new AnnoMatrixObsCrossfilter(annoMatrix, this.obsCrossfilter);
+  }
+
+  /**
+  Selection state - API is identical to ImmutableTypedCrossfilter, as these
+  are just wrappers to lazy create indices.
+  **/
+
+  async select(field, query, spec) {
+    const { annoMatrix } = this;
+    let { obsCrossfilter } = this;
+
+    if (!annoMatrix?._cache?.[field]) {
+      throw new Error("Unknown field name");
+    }
+    if (field === "var") {
+      throw new Error("unable to obsSelect upon the var dimension");
+    }
+
+    // grab the data, so we can grab the index.
+    const df = await annoMatrix.fetch(field, query);
+
+    const dimName = _dimensionNameFromDf(field, df);
+    if (!obsCrossfilter.hasDimension(dimName)) {
+      // lazy index generation - add dimension when first used
+      obsCrossfilter = this._addObsCrossfilterDimension(
+        annoMatrix,
+        obsCrossfilter,
+        field,
+        df
+      );
+    }
+
+    // select
+    obsCrossfilter = obsCrossfilter.select(dimName, spec);
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  selectAll() {
+    /*
+		Select all on any dimension in this field.
+		*/
+    const { annoMatrix } = this;
+    const currentDims = this.obsCrossfilter.dimensionNames();
+    const obsCrossfilter = currentDims.reduce((xfltr, dim) => {
+      return xfltr.select(dim, { mode: "all" });
+    }, this.obsCrossfilter);
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  countSelected() {
+    /* if no data yet indexed in the crossfilter, just say everything is selected */
+    if (this.obsCrossfilter.size() === 0) return this.annoMatrix.nObs;
+    return this.obsCrossfilter.countSelected();
+  }
+
+  allSelectedMask() {
+    /* if no data yet indexed in the crossfilter, just say everything is selected */
+    if (
+      this.obsCrossfilter.size() === 0 ||
+      this.obsCrossfilter.dimensionNames().length === 0
+    ) {
+      /* fake the mask */
+      return new Uint8Array(this.annoMatrix.nObs).fill(1);
+    }
+    return this.obsCrossfilter.allSelectedMask();
+  }
+
+  allSelectedLabels() {
+    /* if no data yet indexed in the crossfilter, just say everything is selected */
+    if (
+      this.obsCrossfilter.size() === 0 ||
+      this.obsCrossfilter.dimensionNames().length === 0
+    ) {
+      return this.annoMatrix.rowIndex.labels();
+    }
+
+    const mask = this.obsCrossfilter.allSelectedMask();
+    const index = this.annoMatrix.rowIndex.isubsetMask(mask);
+    return index.labels();
+  }
+
+  fillByIsSelected(array, selectedValue, deselectedValue) {
+    /* if no data yet indexed in the crossfilter, just say everything is selected */
+    if (
+      this.obsCrossfilter.size() === 0 ||
+      this.obsCrossfilter.dimensionNames().length === 0
+    ) {
+      return array.fill(selectedValue);
+    }
+    return this.obsCrossfilter.fillByIsSelected(
+      array,
+      selectedValue,
+      deselectedValue
+    );
+  }
+
+  /**
+   ** Private below
+   **/
+
+  _addObsCrossfilterDimension(annoMatrix, obsCrossfilter, field, df) {
+    if (field === "var") return obsCrossfilter;
+    const dimName = _dimensionNameFromDf(field, df);
+    const dimParams = this._getObsDimensionParams(field, df);
+    obsCrossfilter = obsCrossfilter.setData(annoMatrix._cache.obs);
+    obsCrossfilter = obsCrossfilter.addDimension(dimName, ...dimParams);
+    return obsCrossfilter;
+  }
+
+  _getColumnBaseType(field, col) {
+    /* Look up the primitive type for this field/col */
+    const colSchema = _getColumnSchema(this.annoMatrix.schema, field, col);
+    return colSchema.type;
+  }
+
+  _getObsDimensionParams(field, df) {
+    /* return the crossfilter dimensiontype type and params for this field/dataframe */
+
+    if (field === "emb") {
+      /* assumed to be 2D */
+      return ["spatial", df.icol(0).asArray(), df.icol(1).asArray()];
+    }
+
+    /* assumed to be 1D */
+    const col = df.icol(0);
+    const colName = df.colIndex.getLabel(0);
+    const type = this._getColumnBaseType(field, colName);
+    if (type === "string" || type === "categorical" || type === "boolean") {
+      return ["enum", col.asArray()];
+    }
+    if (type === "int32") {
+      return ["scalar", col.asArray(), Int32Array];
+    }
+    if (type === "float32") {
+      return ["scalar", col.asArray(), Float32Array];
+    }
+    // Currently not supporting boolean and categorical types.
+    console.error(
+      `Warning - unknown metadata schema (${type}) for field ${field} ${colName}.`
+    );
+    // skip it - we don't know what to do with this type
+
+    return undefined;
+  }
+}

client/src/annoMatrix/fetchHelpers.js

@@ -0,0 +1,27 @@
+export { doBinaryRequest } from "../util/actionHelpers";
+
+/* double URI encode - needed for query-param filters */
+export function _dubEncURIComp(s) {
+  return encodeURIComponent(encodeURIComponent(s));
+}
+
+/* currently unused, consider deleting */
+export function _fetchResult(promise) {
+  let _status = "pending";
+  const res = promise.then(
+    (r) => {
+      _status = "success";
+      return r;
+    },
+    (e) => {
+      _status = "error";
+      throw e;
+    }
+  );
+
+  res.status = () => {
+    return _status;
+  };
+
+  return res;
+}

client/src/annoMatrix/index.js

@@ -0,0 +1,15 @@
+/*
+AnnoMatrix -- Annotated Matrix exported interface
+
+Public API is defined in:
+
+	annoMatrix.js
+	viewCreators.js
+	crossfilter.js
+
+*/
+
+export { default as AnnoMatrixLoader } from "./loader";
+export * from "./viewCreators";
+export { default as AnnoMatrixObsCrossfilter } from "./crossfilter";
+export { default as gcMiddleware } from "./middleware";

client/src/annoMatrix/loader.js

@@ -0,0 +1,307 @@
+import { doBinaryRequest, _dubEncURIComp } from "./fetchHelpers";
+import { matrixFBSToDataframe } from "../util/stateManager/matrix";
+import { _getColumnSchema, _normalizeCategoricalSchema } from "./schema";
+import {
+  addObsAnnoColumn,
+  removeObsAnnoColumn,
+  addObsAnnoCategory,
+  removeObsAnnoCategory,
+  addObsLayout,
+} from "../util/stateManager/schemaHelpers";
+import { isArrayOrTypedArray } from "../util/typeHelpers";
+import { _whereCacheCreate } from "./whereCache";
+import AnnoMatrix from "./annoMatrix";
+import PromiseLimit from "../util/promiseLimit";
+
+const promiseThrottle = new PromiseLimit(5);
+
+export default class AnnoMatrixLoader extends AnnoMatrix {
+  /*
+  AnnoMatrix implementation which proxies to HTTP server using the CXG REST API.
+  Used as the base (non-view) instance.
+
+  Public API is same as AnnoMatrix class (refer there for API description),
+  with the addition of the constructor which bootstraps:
+
+    new AnnoMatrixLoader(serverBaseURL, schema) -> instance
+
+  */
+  constructor(baseURL, schema) {
+    const { nObs, nVar } = schema.dataframe;
+    super(schema, nObs, nVar);
+
+    if (baseURL[baseURL.length - 1] !== "/") {
+      // must have trailing slash
+      baseURL += "/";
+    }
+    this.baseURL = baseURL;
+    Object.seal(this);
+  }
+
+  /**
+   ** Public.  API described in base class.
+   **/
+  addObsAnnoCategory(col, category) {
+    /*
+    Add a new category (aka label) to the schema for an obs column.
+    */
+    const colSchema = _getColumnSchema(this.schema, "obs", col);
+    _writableCategoryTypeCheck(colSchema); // throws on error
+
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.schema = addObsAnnoCategory(this.schema, col, category);
+    return newAnnoMatrix;
+  }
+
+  async removeObsAnnoCategory(col, category, unassignedCategory) {
+    /*
+    Remove a single "category" (aka "label") from the data & schema of an obs column.
+    */
+    const colSchema = _getColumnSchema(this.schema, "obs", col);
+    _writableCategoryTypeCheck(colSchema); // throws on error
+
+    const newAnnoMatrix = await this.resetObsColumnValues(
+      col,
+      category,
+      unassignedCategory
+    );
+    newAnnoMatrix.schema = removeObsAnnoCategory(
+      newAnnoMatrix.schema,
+      col,
+      category
+    );
+    return newAnnoMatrix;
+  }
+
+  dropObsColumn(col) {
+    /*
+		drop column from field
+		*/
+    const colSchema = _getColumnSchema(this.schema, "obs", col);
+    _writableCheck(colSchema); // throws on error
+
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix._cache.obs = this._cache.obs.dropCol(col);
+    newAnnoMatrix.schema = removeObsAnnoColumn(this.schema, col);
+    return newAnnoMatrix;
+  }
+
+  addObsColumn(colSchema, Ctor, value) {
+    /*
+		add a column to field, initializing with value.  Value may 
+    be one of:
+      * an array of values
+      * a primitive type, including null or undefined.
+    If an array, it must be of same size as nObs and same type as Ctor
+		*/
+    colSchema.writable = true;
+    const colName = colSchema.name;
+    if (
+      _getColumnSchema(this.schema, "obs", colName) ||
+      this._cache.obs.hasCol(colName)
+    ) {
+      throw new Error("column already exists");
+    }
+
+    const newAnnoMatrix = this._clone();
+    let data;
+    if (isArrayOrTypedArray(value)) {
+      if (value.constructor !== Ctor)
+        throw new Error("Mismatched value array type");
+      if (value.length !== this.nObs)
+        throw new Error("Value array has incorrect length");
+      data = value.slice();
+    } else {
+      data = new Ctor(this.nObs).fill(value);
+    }
+    newAnnoMatrix._cache.obs = this._cache.obs.withCol(colName, data);
+    _normalizeCategoricalSchema(
+      colSchema,
+      newAnnoMatrix._cache.obs.col(colName)
+    );
+    newAnnoMatrix.schema = addObsAnnoColumn(this.schema, colName, colSchema);
+    return newAnnoMatrix;
+  }
+
+  renameObsColumn(oldCol, newCol) {
+    /*
+    Rename the obs oldColName to newColName.  oldCol must be writable.
+    */
+    const oldColSchema = _getColumnSchema(this.schema, "obs", oldCol);
+    _writableCheck(oldColSchema); // throws on error
+
+    const value = this._cache.obs.hasCol(oldCol)
+      ? this._cache.obs.col(oldCol).asArray()
+      : undefined;
+    return this.dropObsColumn(oldCol).addObsColumn(
+      {
+        ...oldColSchema,
+        name: newCol,
+      },
+      value.constructor,
+      value
+    );
+  }
+
+  async setObsColumnValues(col, rowLabels, value) {
+    /*
+		Set all rows identified by rowLabels to value.
+		*/
+    const colSchema = _getColumnSchema(this.schema, "obs", col);
+    _writableCategoryTypeCheck(colSchema); // throws on error
+
+    // ensure that we have the data in cache before we manipulate it
+    await this.fetch("obs", col);
+    if (!this._cache.obs.hasCol(col))
+      throw new Error("Internal error - user annotation data missing");
+
+    const rowIndices = this.rowIndex.getOffsets(rowLabels);
+    const data = this._cache.obs.col(col).asArray().slice();
+    for (let i = 0, len = rowIndices.length; i < len; i += 1) {
+      const idx = rowIndices[i];
+      if (idx === undefined) throw new Error("Unknown row label");
+      data[idx] = value;
+    }
+
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix._cache.obs = this._cache.obs.replaceColData(col, data);
+    const { categories } = colSchema;
+    if (!categories?.includes(value)) {
+      newAnnoMatrix.schema = addObsAnnoCategory(this.schema, col, value);
+    }
+    return newAnnoMatrix;
+  }
+
+  async resetObsColumnValues(col, oldValue, newValue) {
+    /*
+    Set all rows with value 'oldValue' to 'newValue'.
+    */
+    const colSchema = _getColumnSchema(this.schema, "obs", col);
+    _writableCategoryTypeCheck(colSchema); // throws on error
+
+    if (!colSchema.categories.includes(oldValue)) {
+      throw new Error("unknown category");
+    }
+
+    // ensure that we have the data in cache before we manipulate it
+    await this.fetch("obs", col);
+    if (!this._cache.obs.hasCol(col))
+      throw new Error("Internal error - user annotation data missing");
+
+    const data = this._cache.obs.col(col).asArray().slice();
+    for (let i = 0, l = data.length; i < l; i += 1) {
+      if (data[i] === oldValue) data[i] = newValue;
+    }
+
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix._cache.obs = this._cache.obs.replaceColData(col, data);
+    const { categories } = colSchema;
+    if (!categories?.includes(newValue)) {
+      newAnnoMatrix.schema = addObsAnnoCategory(this.schema, col, newValue);
+    }
+    return newAnnoMatrix;
+  }
+
+  addEmbedding(colSchema) {
+    /*
+    add new layout to the obs embeddings
+    */
+    const { name: colName } = colSchema;
+    if (_getColumnSchema(this.schema, "emb", colName)) {
+      throw new Error("column already exists");
+    }
+
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.schema = addObsLayout(this.schema, colSchema);
+    return newAnnoMatrix;
+  }
+
+  /**
+   ** Private below
+   **/
+  async _doLoad(field, query) {
+    /*
+    _doLoad - evaluates the query against the field. Returns:
+      * whereCache update: column query map mapping the query to the column labels
+      * Dataframe containing the new colums (one per dimension)
+    */
+    let urlQuery;
+    let urlBase;
+    let priority = 10; // default fetch priority
+
+    switch (field) {
+      case "obs":
+      case "var": {
+        urlBase = `${this.baseURL}annotations/${field}`;
+        urlQuery = _encodeQuery("annotation-name", query);
+        break;
+      }
+      case "X": {
+        urlBase = `${this.baseURL}data/var`;
+        urlQuery = _encodeQuery(undefined, query);
+        break;
+      }
+      case "emb": {
+        urlBase = `${this.baseURL}layout/obs`;
+        urlQuery = _encodeQuery("layout-name", query);
+        priority = 0; // high prio load for embeddings
+        break;
+      }
+      default:
+        throw new Error("Unknown field name");
+    }
+
+    const url = `${urlBase}?${urlQuery}`;
+    const buffer = await promiseThrottle.priorityAdd(
+      priority,
+      doBinaryRequest,
+      url
+    );
+    const result = matrixFBSToDataframe(buffer);
+    if (!result || result.isEmpty()) throw Error("Unknown field/col");
+
+    const whereCacheUpdate = _whereCacheCreate(
+      field,
+      query,
+      result.colIndex.labels()
+    );
+
+    if (field === "obs") {
+      /* cough, cough - see comment on method */
+      _normalizeCategoricalSchema(
+        this.schema.annotations.obsByName[query],
+        result.col(query)
+      );
+    }
+
+    return [whereCacheUpdate, result];
+  }
+}
+
+/*
+Utility functions below
+*/
+
+function _encodeQuery(colKey, q) {
+  if (typeof q === "object") {
+    const { field: queryField, column: queryColumn, value: queryValue } = q;
+    return `${_dubEncURIComp(queryField)}:${_dubEncURIComp(
+      queryColumn
+    )}=${_dubEncURIComp(queryValue)}`;
+  }
+  if (!colKey) throw new Error("Unsupported query by name");
+  return `${colKey}=${encodeURIComponent(q)}`;
+}
+
+function _writableCheck(colSchema) {
+  if (!colSchema?.writable) {
+    throw new Error("Unknown or readonly obs column");
+  }
+}
+
+function _writableCategoryTypeCheck(colSchema) {
+  _writableCheck(colSchema);
+  if (colSchema.type !== "categorical") {
+    throw new Error("column must be categorical");
+  }
+}

client/src/annoMatrix/middleware.js

@@ -0,0 +1,66 @@
+/*
+Garbage collection / cache management support
+
+Middleware that knows how to pull annoMatrix from the undoable state,
+and pass it along to the AnnoMatrix class for possible cache GC.
+
+Private interface.
+
+Future work item: this middleware knows internal details of both the
+Undoable metareducer and the AnnoMatrix private API.  It would be helpful
+to make the Undoable interface better factored.
+*/
+
+const annoMatrixGC = (store) => (next) => (action) => {
+  if (_itIsTimeForGC()) {
+    _doGC(store);
+  }
+  return next(action);
+};
+
+let lastGCTime = 0;
+const InterGCDelayMS = 30 * 1000; // 30 seconds
+function _itIsTimeForGC() {
+  /*
+  we don't want to run GC on every dispatch, so throttle it a bit.
+
+  Runs every InterGCDelay period
+  */
+  const now = Date.now();
+  if (now - lastGCTime > InterGCDelayMS) {
+    lastGCTime = now;
+    return true;
+  }
+  return false;
+}
+
+function _doGC(store) {
+  const state = store.getState();
+
+  // these should probably be a function imported from undoable.js, etc, as
+  // they have overly intimiate knowledge of our reducers.
+  const undoablePast = state["@@undoable/past"];
+  const undoableFuture = state["@@undoable/future"];
+  const undoableStack = undoablePast
+    .concat(undoableFuture)
+    .flatMap((snapshot) =>
+      snapshot.filter((v) => v[0] === "annoMatrix").map((v) => v[1])
+    );
+  const currentAnnoMatrix = state.annoMatrix;
+
+  /*
+  We want to identify those matrixes currently "hot", ie, linked from the current annoMatrix,
+  as our current gc algo is more aggressive with those not hot.
+  */
+  const allAnnoMatrices = new Map(
+    undoableStack.map((m) => [m, { isHot: false }])
+  );
+  let am = currentAnnoMatrix;
+  while (am) {
+    allAnnoMatrices.set(am, { isHot: true });
+    am = am.viewOf;
+  }
+  allAnnoMatrices.forEach((hints, annoMatrix) => annoMatrix._gc(hints));
+}
+
+export default annoMatrixGC;

client/src/annoMatrix/schema.js

@@ -0,0 +1,88 @@
+/*
+Private helper functions related to schema
+*/
+import catLabelSort from "../util/catLabelSort";
+import { unassignedCategoryLabel } from "../globals";
+
+export function _getColumnSchema(schema, field, col) {
+  /* look up the column definition */
+  switch (field) {
+    case "obs":
+      if (typeof col === "object")
+        throw new Error("unable to get column schema by query");
+      return schema.annotations.obsByName[col];
+    case "var":
+      if (typeof col === "object")
+        throw new Error("unable to get column schema by query");
+      return schema.annotations.varByName[col];
+    case "emb":
+      if (typeof col === "object")
+        throw new Error("unable to get column schema by query");
+      return schema.layout.obsByName[col];
+    case "X":
+      return schema.dataframe;
+    default:
+      throw new Error(`unknown field name: ${field}`);
+  }
+}
+
+export function _getColumnDimensionNames(schema, field, col) {
+  /*
+		field/col may be an alias for multiple columns. Currently used to map ND 
+		values to 1D dataframe columns for embeddings/layout. Signfied by the presence
+		of the "dims" value in the schema.
+		*/
+  const colSchema = _getColumnSchema(schema, field, col);
+  if (!colSchema) {
+    return undefined;
+  }
+  return colSchema.dims || [col];
+}
+
+export function _schemaColumns(schema, field) {
+  switch (field) {
+    case "obs":
+      return Object.keys(schema.annotations.obsByName);
+    case "var":
+      return Object.keys(schema.annotations.varByName);
+    case "emb":
+      return Object.keys(schema.layout.obsByName);
+    default:
+      throw new Error(`unknown field name: ${field}`);
+  }
+}
+
+export function _getWritableColumns(schema, field) {
+  if (field !== "obs") return [];
+  return schema.annotations.obs.columns
+    .filter((v) => v.writable)
+    .map((v) => v.name);
+}
+
+export function _isContinuousType(schema) {
+  const { type } = schema;
+  return !(type === "string" || type === "boolean" || type === "categorical");
+}
+
+export function _normalizeCategoricalSchema(colSchema, col) {
+  const { type, writable } = colSchema;
+  if (
+    type === "string" ||
+    type === "boolean" ||
+    type === "categorical" ||
+    writable
+  ) {
+    const categorySet = new Set(
+      col.summarizeCategorical().categories.concat(colSchema.categories ?? [])
+    );
+    if (writable && !categorySet.has(unassignedCategoryLabel)) {
+      categorySet.add(unassignedCategoryLabel);
+    }
+    colSchema.categories = Array.from(categorySet);
+  }
+
+  if (colSchema.categories) {
+    colSchema.categories = catLabelSort(writable, colSchema.categories);
+  }
+  return colSchema;
+}

client/src/annoMatrix/viewCreators.js

@@ -0,0 +1,70 @@
+/*
+View creators.  These are helper functions which create new views from existing
+instances of AnnoMatrix, implementing common UI functions.
+*/
+
+import { AnnoMatrixRowSubsetView, AnnoMatrixClipView } from "./views";
+
+export function isubsetMask(annoMatrix, obsMask) {
+  /*
+		Subset annomatrix to contain the rows which have truish value in the mask.
+    Maks length must equal annoMatrix.nObs (row count).
+	*/
+  return isubset(annoMatrix, _maskToList(obsMask));
+}
+
+export function isubset(annoMatrix, obsOffsets) {
+  /*
+		Subset annomatrix to contain the positions contained in the obsOffsets array
+
+    Example:
+
+      isubset(annoMatrix, [0, 1]) -> annoMatrix with only the first two rows
+	*/
+  const obsIndex = annoMatrix.rowIndex.isubset(obsOffsets);
+  return new AnnoMatrixRowSubsetView(annoMatrix, obsIndex);
+}
+
+export function subset(annoMatrix, obsLabels) {
+  /*
+		subset based on labels
+  */
+  const obsIndex = annoMatrix.rowIndex.subset(obsLabels);
+  return new AnnoMatrixRowSubsetView(annoMatrix, obsIndex);
+}
+
+export function subsetByIndex(annoMatrix, obsIndex) {
+  /*
+  subset based upon the new obs index.
+  */
+  return new AnnoMatrixRowSubsetView(annoMatrix, obsIndex);
+}
+
+export function clip(annoMatrix, qmin, qmax) {
+  /*
+		Create a view that clips all continuous data to the [min, max] range.
+		The matrix shape does not change, but the continuous values outside the
+		specified range will become a NaN.
+		*/
+  return new AnnoMatrixClipView(annoMatrix, qmin, qmax);
+}
+
+/*
+Private utility functions below
+*/
+
+function _maskToList(mask) {
+  /* convert masks to lists - method wastes space, but is fast */
+  if (!mask) {
+    return null;
+  }
+  const list = new Int32Array(mask.length);
+  let elems = 0;
+  for (let i = 0, l = mask.length; i < l; i += 1) {
+    if (mask[i]) {
+      list[elems] = i;
+      elems += 1;
+    }
+  }
+  return list.subarray(0, elems);
+}

client/src/annoMatrix/views.js

@@ -0,0 +1,176 @@
+/* eslint-disable max-classes-per-file -- Classes are interrelated*/
+
+/*
+Views on the annomatrix.  all API here is defined in viewCreators.js and annoMatrix.js.
+*/
+import clip from "../util/clip";
+import AnnoMatrix from "./annoMatrix";
+import { _whereCacheCreate } from "./whereCache";
+import { _isContinuousType, _getColumnSchema } from "./schema";
+
+class AnnoMatrixView extends AnnoMatrix {
+  constructor(viewOf, rowIndex = null) {
+    const nObs = rowIndex ? rowIndex.size() : viewOf.nObs;
+    super(viewOf.schema, nObs, viewOf.nVar, rowIndex || viewOf.rowIndex);
+    this.viewOf = viewOf;
+    this.isView = true;
+  }
+
+  addObsAnnoCategory(col, category) {
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.viewOf = this.viewOf.addObsAnnoCategory(col, category);
+    newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema;
+    return newAnnoMatrix;
+  }
+
+  async removeObsAnnoCategory(col, category, unassignedCategory) {
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.viewOf = await this.viewOf.removeObsAnnoCategory(
+      col,
+      category,
+      unassignedCategory
+    );
+    newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema;
+    return newAnnoMatrix;
+  }
+
+  dropObsColumn(col) {
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.viewOf = this.viewOf.dropObsColumn(col);
+    newAnnoMatrix._cache.obs = this._cache.obs.dropCol(col);
+    newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema;
+    return newAnnoMatrix;
+  }
+
+  addObsColumn(colSchema, Ctor, value) {
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.viewOf = this.viewOf.addObsColumn(colSchema, Ctor, value);
+    newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema;
+    return newAnnoMatrix;
+  }
+
+  renameObsColumn(oldCol, newCol) {
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.viewOf = this.viewOf.renameObsColumn(oldCol, newCol);
+    newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema;
+    return newAnnoMatrix;
+  }
+
+  async setObsColumnValues(col, rowLabels, value) {
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.viewOf = await this.viewOf.setObsColumnValues(
+      col,
+      rowLabels,
+      value
+    );
+    newAnnoMatrix._cache.obs = this._cache.obs.dropCol(col);
+    newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema;
+    return newAnnoMatrix;
+  }
+
+  async resetObsColumnValues(col, oldValue, newValue) {
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.viewOf = await this.viewOf.resetObsColumnValues(
+      col,
+      oldValue,
+      newValue
+    );
+    newAnnoMatrix._cache.obs = this._cache.obs.dropCol(col);
+    newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema;
+    return newAnnoMatrix;
+  }
+
+  addEmbedding(colSchema) {
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.viewOf = this.viewOf.addEmbedding(colSchema);
+    newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema;
+    return newAnnoMatrix;
+  }
+}
+
+class AnnoMatrixMapView extends AnnoMatrixView {
+  /*
+	A view which knows how to transform its data.
+	*/
+  constructor(viewOf, mapFn) {
+    super(viewOf);
+    this.mapFn = mapFn;
+  }
+
+  async _doLoad(field, query) {
+    const df = await this.viewOf._fetch(field, query);
+    const dfMapped = df.mapColumns((colData, colIdx) => {
+      const colLabel = df.colIndex.getLabel(colIdx);
+      const colSchema = _getColumnSchema(this.schema, field, colLabel);
+      return this.mapFn(field, colLabel, colSchema, colData, df);
+    });
+    const whereCacheUpdate = _whereCacheCreate(
+      field,
+      query,
+      dfMapped.colIndex.labels()
+    );
+    return [whereCacheUpdate, dfMapped];
+  }
+}
+
+export class AnnoMatrixClipView extends AnnoMatrixMapView {
+  /*
+	A view which is a clipped transformation of its parent
+	*/
+  constructor(viewOf, qmin, qmax) {
+    super(viewOf, (field, colLabel, colSchema, colData, df) =>
+      _clipAnnoMatrix(field, colLabel, colSchema, colData, df, qmin, qmax)
+    );
+    this.isClipped = true;
+    this.clipRange = [qmin, qmax];
+    Object.seal(this);
+  }
+}
+
+export class AnnoMatrixRowSubsetView extends AnnoMatrixView {
+  /*
+	A view which is a subset of total rows.
+	*/
+  constructor(viewOf, rowIndex) {
+    super(viewOf, rowIndex);
+    Object.seal(this);
+  }
+
+  async _doLoad(field, query) {
+    const df = await this.viewOf._fetch(field, query);
+
+    // don't try to row-subset the var dimension.
+    if (field === "var") {
+      return [null, df];
+    }
+
+    const dfSubset = df.subset(null, null, this.rowIndex);
+    const whereCacheUpdate = _whereCacheCreate(
+      field,
+      query,
+      dfSubset.colIndex.labels()
+    );
+    return [whereCacheUpdate, dfSubset];
+  }
+}
+
+/*
+Utility functions below
+*/
+
+function _clipAnnoMatrix(field, colLabel, colSchema, colData, df, qmin, qmax) {
+  /* only clip obs and var scalar columns */
+  if (field !== "obs" && field !== "X") return colData;
+  if (!_isContinuousType(colSchema)) return colData;
+  if (qmin < 0) qmin = 0;
+  if (qmax > 1) qmax = 1;
+  if (qmin === 0 && qmax === 1) return colData;
+
+  const quantiles = df.col(colLabel).summarize().percentiles;
+  const lower = quantiles[100 * qmin];
+  const upper = quantiles[100 * qmax];
+  const clippedData = clip(colData.slice(), lower, upper, Number.NaN);
+  return clippedData;
+}
+
+/* eslint-enable max-classes-per-file -- enable*/

client/src/annoMatrix/whereCache.js

@@ -0,0 +1,92 @@
+/*
+Private support functions.
+
+Support for a "where" query, eg,
+
+	{ where: { field: "var", column: "gene", value: "FOXP2" }}
+
+These evaluate to a given column label.
+
+The "where cache" is a map that saves evaluated queries and points 
+to the column label they resolve to.
+
+Data structure, using X as the example field being queried, and var as
+the index.
+
+{ 
+	X: {
+		var: Map(
+			column_label_in_var => Map(value_in_var_column => [column_label_in_X, ...])
+		)
+	}
+}
+*/
+import { _getColumnDimensionNames } from "./schema";
+
+export function _whereCacheGet(whereCache, schema, field, query) {
+  /* 
+	query will either be an where query (object) or a column name (string).
+
+	Return array of column labels or undefined.
+	*/
+
+  if (typeof query === "object") {
+    const { field: queryField, column: queryColumn, value: queryValue } = query;
+
+    const columnMap = whereCache?.[field]?.[queryField];
+    if (columnMap === undefined) return [undefined];
+
+    const valueMap = columnMap.get(queryColumn);
+    if (valueMap === undefined) return [undefined];
+
+    const columnLabels = valueMap.get(queryValue);
+    return columnLabels === undefined ? [undefined] : columnLabels;
+  }
+
+  const colDims = _getColumnDimensionNames(schema, field, query);
+  return colDims === undefined ? [undefined] : colDims;
+}
+
+export function _whereCacheCreate(field, query, columnLabels) {
+  /*
+	Create a new whereCache
+	*/
+  if (typeof query !== "object") return null;
+
+  const { field: queryField, column: queryColumn, value: queryValue } = query;
+  const whereCache = {
+    [field]: {
+      [queryField]: new Map([
+        [queryColumn, new Map([[queryValue, columnLabels]])],
+      ]),
+    },
+  };
+  return whereCache;
+}
+
+function __whereCacheMerge(dst, src) {
+  /*
+	merge src into dst (modifies dst)
+	*/
+  if (!dst) dst = {};
+  if (!src || typeof src !== "object") return dst;
+  Object.entries(src).forEach(([field, query]) => {
+    if (!Object.prototype.hasOwnProperty.call(dst, field)) dst[field] = {};
+    Object.entries(query).forEach(([queryField, columnMap]) => {
+      if (!Object.prototype.hasOwnProperty.call(dst[field], queryField))
+        dst[field][queryField] = new Map();
+      columnMap.forEach((valueMap, queryColumn) => {
+        if (!dst[field][queryField].has(queryColumn))
+          dst[field][queryField].set(queryColumn, new Map());
+        valueMap.forEach((columnLabels, queryValue) => {
+          dst[field][queryField].get(queryColumn).set(queryValue, columnLabels);
+        });
+      });
+    });
+  });
+  return dst;
+}
+
+export function _whereCacheMerge(...caches) {
+  return caches.reduce((dst, src) => __whereCacheMerge(dst, src), {});
+}

client/src/components/app.js

@@ -4,26 +4,24 @@ import Helmet from "react-helmet";
 import { connect } from "react-redux";
 
 import Container from "./framework/container";
+import Layout from "./framework/layout";
 import LeftSideBar from "./leftSidebar";
 import RightSideBar from "./rightSidebar";
 import Legend from "./continuousLegend";
 import Graph from "./graph/graph";
 import MenuBar from "./menubar";
 import Autosave from "./autosave";
+import Embedding from "./embedding";
+import TermsOfServicePrompt from "./termsPrompt";
 
 import actions from "../actions";
 
-@connect(state => ({
+@connect((state) => ({
   loading: state.controls.loading,
   error: state.controls.error,
-  graphRenderCounter: state.controls.graphRenderCounter
+  graphRenderCounter: state.controls.graphRenderCounter,
 }))
 class App extends React.Component {
-  constructor(props) {
-    super(props);
-    this.state = {};
-  }
-
   componentDidMount() {
     const { dispatch } = this.props;
 
@@ -32,24 +30,7 @@ class App extends React.Component {
     this._onURLChanged();
 
     dispatch(actions.doInitialDataLoad(window.location.search));
-
-    /* listen for resize events */
-    window.addEventListener("resize", () => {
-      dispatch({
-        type: "window resize",
-        data: {
-          height: window.innerHeight,
-          width: window.innerWidth
-        }
-      });
-    });
-    dispatch({
-      type: "window resize",
-      data: {
-        height: window.innerHeight,
-        width: window.innerWidth
-      }
-    });
+    this.forceUpdate();
   }
 
   _onURLChanged() {
@@ -69,7 +50,7 @@ class App extends React.Component {
               position: "fixed",
               fontWeight: 500,
               top: window.innerHeight / 2,
-              left: window.innerWidth / 2 - 50
+              left: window.innerWidth / 2 - 50,
             }}
           >
             loading cellxgene
@@ -81,20 +62,28 @@ class App extends React.Component {
               position: "fixed",
               fontWeight: 500,
               top: window.innerHeight / 2,
-              left: window.innerWidth / 2 - 50
+              left: window.innerWidth / 2 - 50,
             }}
           >
-            error loading
+            error loading cellxgene
           </div>
         ) : null}
-        <div>
-          {loading ? null : <LeftSideBar />}
-          {loading ? null : <RightSideBar />}
-          {loading ? null : <MenuBar />}
-          {loading ? null : <Graph key={graphRenderCounter} />}
-          {loading ? null : <Autosave />}
-          <Legend />
-        </div>
+        {loading || error ? null : (
+          <Layout>
+            <LeftSideBar />
+            {(viewportRef) => (
+              <>
+                <MenuBar />
+                <Embedding />
+                <Autosave />
+                <TermsOfServicePrompt />
+                <Legend viewportRef={viewportRef} />
+                <Graph key={graphRenderCounter} viewportRef={viewportRef} />
+              </>
+            )}
+            <RightSideBar />
+          </Layout>
+        )}
       </Container>
     );
   }

client/src/components/autosave/filenameDialog.js

@@ -3,28 +3,26 @@ import { connect } from "react-redux";
 
 import {
   Button,
-  Tooltip,
-  InputGroup,
-  Dialog,
   Classes,
-  Colors
+  Code,
+  Colors,
+  Dialog,
+  InputGroup,
+  Tooltip,
 } from "@blueprintjs/core";
 
-@connect(state => ({
-  universe: state.universe,
+@connect((state) => ({
   idhash: state.config?.parameters?.["annotations-user-data-idhash"] ?? null,
   annotations: state.annotations,
-  obsAnnotations: state.universe.obsAnnotations,
-  saveInProgress: state.autosave?.saveInProgress ?? false,
-  lastSavedObsAnnotations: state.autosave?.lastSavedObsAnnotations,
-  error: state.autosave?.error,
-  writableCategoriesEnabled: state.config?.parameters?.["annotations"] ?? false
+  auth: state.config?.authentication,
+  userInfo: state.userInfo,
+  writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
 }))
 class FilenameDialog extends React.Component {
   constructor(props) {
     super(props);
     this.state = {
-      filenameText: ""
+      filenameText: "",
     };
   }
 
@@ -36,7 +34,7 @@ class FilenameDialog extends React.Component {
 
     dispatch({
       type: "set annotations collection name",
-      data: filenameText
+      data: filenameText,
     });
   };
 
@@ -71,10 +69,10 @@ class FilenameDialog extends React.Component {
             fontStyle: "italic",
             fontSize: 12,
             marginTop: 5,
-            color: Colors.ORANGE3
+            color: Colors.ORANGE3,
           }}
         >
-          {"Name cannot be blank"}
+          Name cannot be blank
         </span>
       );
     } else if (err === "characters") {
@@ -84,10 +82,10 @@ class FilenameDialog extends React.Component {
             fontStyle: "italic",
             fontSize: 12,
             marginTop: 5,
-            color: Colors.ORANGE3
+            color: Colors.ORANGE3,
           }}
         >
-          {"Only alphanumeric and underscore allowed"}
+          Only alphanumeric and underscore allowed
         </span>
       );
     }
@@ -95,12 +93,19 @@ class FilenameDialog extends React.Component {
   };
 
   render() {
-    const { writableCategoriesEnabled, annotations, idhash } = this.props;
+    const {
+      writableCategoriesEnabled,
+      annotations,
+      idhash,
+      userInfo,
+    } = this.props;
     const { filenameText } = this.state;
 
     return writableCategoriesEnabled &&
+      annotations.promptForFilename &&
       !annotations.dataCollectionNameIsReadOnly &&
-      !annotations.dataCollectionName ? (
+      !annotations.dataCollectionName &&
+      userInfo.is_authenticated ? (
       <Dialog
         icon="tag"
         title="Annotations Collection"
@@ -108,20 +113,23 @@ class FilenameDialog extends React.Component {
         onClose={this.dismissFilenameDialog}
       >
         <form
-          onSubmit={e => {
+          onSubmit={(e) => {
             e.preventDefault();
             this.handleCreateFilename();
           }}
         >
-          <div className={Classes.DIALOG_BODY}>
+          <div className={Classes.DIALOG_BODY} data-testid="annotation-dialog">
             <div style={{ marginBottom: 20 }}>
               <p>Name your annotations collection:</p>
               <InputGroup
                 autoFocus
                 value={filenameText}
                 intent={this.filenameError(filenameText) ? "warning" : "none"}
-                onChange={e => this.setState({ filenameText: e.target.value })}
+                onChange={(e) =>
+                  this.setState({ filenameText: e.target.value })
+                }
                 leftIcon="tag"
+                data-testid="new-annotation-name"
               />
               <p
                 style={{
@@ -129,7 +137,7 @@ class FilenameDialog extends React.Component {
                   visibility: this.filenameError(filenameText)
                     ? "visible"
                     : "hidden",
-                  color: Colors.ORANGE3
+                  color: Colors.ORANGE3,
                 }}
               >
                 {this.filenameErrorMessage(filenameText)}
@@ -138,9 +146,9 @@ class FilenameDialog extends React.Component {
             <div>
               <p>
                 Your annotations are stored in this file:
-                <code className="bp3-code">
+                <Code>
                   {filenameText}-{idhash}.csv
-                </code>
+                </Code>
               </p>
               <p style={{ fontStyle: "italic" }}>
                 (We added a unique ID to your filename)
@@ -157,6 +165,7 @@ class FilenameDialog extends React.Component {
                 onClick={this.handleCreateFilename}
                 intent="primary"
                 type="submit"
+                data-testid="submit-annotation"
               >
                 Create annotations collection
               </Button>

client/src/components/autosave/index.js

@@ -1,23 +1,21 @@
 import React from "react";
 import { connect } from "react-redux";
-import * as globals from "../../globals";
 import actions from "../../actions";
 import FilenameDialog from "./filenameDialog";
 
-@connect(state => ({
-  universe: state.universe,
+@connect((state) => ({
   annotations: state.annotations,
-  obsAnnotations: state.universe.obsAnnotations,
   saveInProgress: state.autosave?.saveInProgress ?? false,
-  lastSavedObsAnnotations: state.autosave?.lastSavedObsAnnotations,
   error: state.autosave?.error,
-  writableCategoriesEnabled: state.config?.parameters?.["annotations"] ?? false
+  writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
+  annoMatrix: state.annoMatrix,
+  lastSavedAnnoMatrix: state.autosave?.lastSavedAnnoMatrix,
 }))
 class Autosave extends React.Component {
   constructor(props) {
     super(props);
     this.state = {
-      timer: null
+      timer: null,
     };
   }
 
@@ -42,16 +40,14 @@ class Autosave extends React.Component {
   tick = () => {
     const { dispatch, saveInProgress } = this.props;
     if (this.needToSave() && !saveInProgress) {
-      dispatch(actions.saveObsAnnotations());
+      dispatch(actions.saveObsAnnotationsAction());
     }
   };
 
   needToSave = () => {
     /* return true if we need to save, false if we don't */
-    const { obsAnnotations, lastSavedObsAnnotations } = this.props;
-    return (
-      lastSavedObsAnnotations && obsAnnotations !== lastSavedObsAnnotations
-    );
+    const { annoMatrix, lastSavedAnnoMatrix } = this.props;
+    return actions.needToSaveObsAnnotations(annoMatrix, lastSavedAnnoMatrix);
   };
 
   statusMessage() {
@@ -63,21 +59,37 @@ class Autosave extends React.Component {
   }
 
   render() {
-    const { writableCategoriesEnabled } = this.props;
-    return writableCategoriesEnabled ? (
+    const {
+      writableCategoriesEnabled,
+      saveInProgress,
+      lastSavedAnnoMatrix,
+    } = this.props;
+    const initialDataLoadComplete = lastSavedAnnoMatrix;
+
+    if (!writableCategoriesEnabled) return null;
+
+    return (
       <div
         id="autosave"
+        data-testclass={
+          !initialDataLoadComplete
+            ? "autosave-init"
+            : this.needToSave() || saveInProgress
+            ? "autosave-incomplete"
+            : "autosave-complete"
+        }
         style={{
-          position: "fixed",
+          position: "absolute",
           display: "inherit",
-          right: globals.leftSidebarWidth + 5,
-          bottom: 5
+          right: 8,
+          bottom: 8,
+          zIndex: 1,
         }}
       >
         {this.statusMessage()}
         <FilenameDialog />
       </div>
-    ) : null;
+    );
   }
 }